diff --git a/crmd/callbacks.c b/crmd/callbacks.c
index 6b6504108f..4c2b3a4912 100644
--- a/crmd/callbacks.c
+++ b/crmd/callbacks.c
@@ -1,309 +1,309 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <crm/crm.h>
#include <string.h>
#include <crmd_fsa.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/cluster.h>
#include <crm/cib.h>
#include <crmd.h>
#include <crmd_messages.h>
#include <crmd_callbacks.h>
#include <crmd_lrm.h>
#include <tengine.h>
#include <membership.h>
void crmd_ha_connection_destroy(gpointer user_data);
/* From join_dc... */
extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
void
crmd_ha_connection_destroy(gpointer user_data)
{
crm_trace("Invoked");
if (is_set(fsa_input_register, R_HA_DISCONNECTED)) {
/* we signed out, so this is expected */
crm_info("Heartbeat disconnection complete");
return;
}
crm_crit("Lost connection to heartbeat service!");
register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL);
trigger_fsa(fsa_source);
}
void
crmd_ha_msg_filter(xmlNode * msg)
{
if (AM_I_DC) {
const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
if (safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
const char *from = crm_element_value(msg, F_ORIG);
if (safe_str_neq(from, fsa_our_uname)) {
int level = LOG_INFO;
const char *op = crm_element_value(msg, F_CRM_TASK);
/* make sure the election happens NOW */
if (fsa_state != S_ELECTION) {
ha_msg_input_t new_input;
level = LOG_WARNING;
new_input.msg = msg;
register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
__FUNCTION__);
}
do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
goto done;
}
}
} else {
const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
if (safe_str_eq(sys_to, CRM_SYSTEM_DC)) {
return;
}
}
/* crm_log_xml_trace("HA[inbound]", msg); */
route_message(C_HA_MESSAGE, msg);
done:
trigger_fsa(fsa_source);
}
#define state_text(state) ((state)? (const char *)(state) : "in unknown state")
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
uint32_t old = 0;
uint32_t changed = 0;
bool appeared = FALSE;
bool is_remote = is_set(node->flags, crm_remote_node);
const char *status = NULL;
/* Crmd waits to receive some information from the membership layer before
* declaring itself operational. If this is being called for a cluster node,
* indicate that we have it.
*/
if (!is_remote) {
set_bit(fsa_input_register, R_PEER_DATA);
}
if (node->uname == NULL) {
return;
}
switch (type) {
case crm_status_uname:
- /* If we've never seen the node, then it also wont be in the status section */
+ /* If we've never seen the node, then it also won't be in the status section */
crm_info("%s is now %s", node->uname, state_text(node->state));
return;
case crm_status_rstate:
case crm_status_nstate:
/* This callback should not be called unless the state actually
* changed, but here's a failsafe just in case.
*/
CRM_CHECK(safe_str_neq(data, node->state), return);
crm_info("%s node %s is now %s (was %s)",
(is_remote? "Remote" : "Cluster"),
node->uname, state_text(node->state), state_text(data));
if (safe_str_eq(CRM_NODE_MEMBER, node->state)) {
appeared = TRUE;
if (!is_remote) {
remove_stonith_cleanup(node->uname);
}
}
crmd_notify_node_event(node);
break;
case crm_status_processes:
if (data) {
old = *(const uint32_t *)data;
changed = node->processes ^ old;
}
status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
node->uname, peer2text(proc_flags), status,
AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);
if ((changed & proc_flags) == 0) {
/* Peer process did not change */
crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
return;
} else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
crm_trace("Not connected");
return;
} else if (fsa_state == S_STOPPING) {
crm_trace("Stopping");
return;
}
appeared = (node->processes & proc_flags) != 0;
if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
/* Did we get evicted? */
crm_notice("Our peer connection failed");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
} else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
/* Did the DC leave us? */
crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
}
} else if(AM_I_DC && appeared == FALSE) {
crm_info("Peer %s left us", node->uname);
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
/* crm_update_peer_join(__FUNCTION__, node, crm_join_none); */
}
break;
}
if (AM_I_DC) {
xmlNode *update = NULL;
int flags = node_update_peer;
gboolean alive = is_remote? appeared : crm_is_peer_active(node);
crm_action_t *down = match_down_event(node->uuid, appeared);
crm_trace("Alive=%d, appeared=%d, down=%d",
alive, appeared, (down? down->id : -1));
if (alive && type == crm_status_processes) {
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
if (down) {
const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
if (safe_str_eq(task, CRM_OP_FENCE)) {
/* tengine_stonith_callback() confirms fence actions */
crm_trace("Updating CIB %s stonithd reported fencing of %s complete",
(down->confirmed? "after" : "before"), node->uname);
} else if ((alive == FALSE) && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
crm_notice("%s of peer %s is complete "CRM_XS" op=%d",
task, node->uname, down->id);
/* down->confirmed = TRUE; */
stop_te_timer(down->timer);
if (!is_remote) {
flags |= node_update_join | node_update_expected;
crmd_peer_down(node, FALSE);
check_join_state(fsa_state, __FUNCTION__);
}
update_graph(transition_graph, down);
trigger_graph();
} else {
crm_trace("Node %s is %salive, was expected to %s (op %d)",
node->uname, (alive? "" : "not "), task, down->id);
}
} else if (appeared == FALSE) {
crm_notice("Stonith/shutdown of %s not matched", node->uname);
if (!is_remote) {
crm_update_peer_join(__FUNCTION__, node, crm_join_none);
check_join_state(fsa_state, __FUNCTION__);
}
abort_transition(INFINITY, tg_restart, "Node failure", NULL);
fail_incompletable_actions(transition_graph, node->uuid);
} else {
crm_trace("Node %s came up, was not expected to be down",
node->uname);
}
if (is_remote) {
/* A pacemaker_remote node won't have its cluster status updated
* in the CIB by membership-layer callbacks, so do it here.
*/
flags |= node_update_cluster;
/* Trigger resource placement on newly integrated nodes */
if (appeared) {
abort_transition(INFINITY, tg_restart,
"pacemaker_remote node integrated", NULL);
}
}
/* Update the CIB node state */
update = create_node_state_update(node, flags, NULL, __FUNCTION__);
fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(update);
}
trigger_fsa(fsa_source);
}
void
crmd_cib_connection_destroy(gpointer user_data)
{
CRM_CHECK(user_data == fsa_cib_conn,;);
crm_trace("Invoked");
trigger_fsa(fsa_source);
fsa_cib_conn->state = cib_disconnected;
if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
crm_info("Connection to the CIB terminated...");
return;
}
/* eventually this will trigger a reconnect, not a shutdown */
crm_err("Connection to the CIB terminated...");
register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
clear_bit(fsa_input_register, R_CIB_CONNECTED);
return;
}
gboolean
crm_fsa_trigger(gpointer user_data)
{
crm_trace("Invoked (queue len: %d)", g_list_length(fsa_message_queue));
s_crmd_fsa(C_FSA_INTERNAL);
crm_trace("Exited (queue len: %d)", g_list_length(fsa_message_queue));
return TRUE;
}
diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c
index 29001214b8..92cc7b8322 100644
--- a/crmd/te_callbacks.c
+++ b/crmd/te_callbacks.c
@@ -1,847 +1,847 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/stat.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/msg_xml.h>
#include <tengine.h>
#include <te_callbacks.h>
#include <crmd_fsa.h>
#include <crm/cluster.h> /* For ONLINESTATUS etc */
void te_update_confirm(const char *event, xmlNode * msg);
extern char *te_uuid;
gboolean shuttingdown = FALSE;
crm_graph_t *transition_graph;
crm_trigger_t *transition_trigger = NULL;
/* #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */
#define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']"
static const char *
get_node_id(xmlNode * rsc_op)
{
xmlNode *node = rsc_op;
while (node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) {
node = node->parent;
}
CRM_CHECK(node != NULL, return NULL);
return ID(node);
}
static void
te_legacy_update_diff(const char *event, xmlNode * diff)
{
int lpc, max;
xmlXPathObject *xpathObj = NULL;
CRM_CHECK(diff != NULL, return);
xml_log_patchset(LOG_TRACE, __FUNCTION__, diff);
if (cib_config_changed(NULL, NULL, &diff)) {
abort_transition(INFINITY, tg_restart, "Non-status change", diff);
goto bail; /* configuration changed */
}
/* Tickets Attributes - Added/Updated */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
if (numXpathResults(xpathObj) > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
goto bail;
}
freeXpathObject(xpathObj);
/* Tickets Attributes - Removed */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
if (numXpathResults(xpathObj) > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
goto bail;
}
freeXpathObject(xpathObj);
/* Transient Attributes - Added/Updated */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
XML_TAG_TRANSIENT_NODEATTRS "//" XML_CIB_TAG_NVPAIR);
max = numXpathResults(xpathObj);
for (lpc = 0; lpc < max; lpc++) {
xmlNode *attr = getXpathResult(xpathObj, lpc);
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
const char *value = NULL;
if (safe_str_eq(CRM_OP_PROBED, name)) {
value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
}
if (crm_is_true(value) == FALSE) {
abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
crm_log_xml_trace(attr, "Abort");
goto bail;
}
}
freeXpathObject(xpathObj);
/* Transient Attributes - Removed */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
XML_TAG_TRANSIENT_NODEATTRS);
if (numXpathResults(xpathObj) > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
goto bail;
}
freeXpathObject(xpathObj);
/*
* Check for and fast-track the processing of LRM refreshes
* In large clusters this can result in _huge_ speedups
*
* Unfortunately we can only do so when there are no pending actions
* Otherwise we could miss updates we're waiting for and stall
*
*/
xpathObj = NULL;
if (transition_graph->pending == 0) {
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
XML_LRM_TAG_RESOURCE);
}
max = numXpathResults(xpathObj);
if (max > 1) {
/* Updates by, or in response to, TE actions will never contain updates
* for more than one resource at a time
*/
crm_debug("Detected LRM refresh - %d resources updated: Skipping all resource events", max);
crm_log_xml_trace(diff, "lrm-refresh");
abort_transition(INFINITY, tg_restart, "LRM Refresh", NULL);
goto bail;
}
freeXpathObject(xpathObj);
/* Process operation updates */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
if (numXpathResults(xpathObj)) {
/*
<status>
<node_state id="node1" state=CRMD_JOINSTATE_MEMBER exp_state="active">
<lrm>
<lrm_resources>
<rsc_state id="" rsc_id="rsc4" node_id="node1" rsc_state="stopped"/>
*/
int lpc = 0, max = numXpathResults(xpathObj);
for (lpc = 0; lpc < max; lpc++) {
xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
const char *node = get_node_id(rsc_op);
process_graph_event(rsc_op, node);
}
}
freeXpathObject(xpathObj);
/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
max = numXpathResults(xpathObj);
for (lpc = 0; lpc < max; lpc++) {
int path_max = 0;
const char *op_id = NULL;
char *rsc_op_xpath = NULL;
xmlXPathObject *op_match = NULL;
xmlNode *match = getXpathResult(xpathObj, lpc);
CRM_LOG_ASSERT(match != NULL);
if(match == NULL) { continue; };
op_id = ID(match);
path_max = strlen(rsc_op_template) + strlen(op_id) + 1;
rsc_op_xpath = calloc(1, path_max);
snprintf(rsc_op_xpath, path_max, rsc_op_template, op_id);
op_match = xpath_search(diff, rsc_op_xpath);
if (numXpathResults(op_match) == 0) {
/* Prevent false positives by matching cancelations too */
const char *node = get_node_id(match);
crm_action_t *cancelled = get_cancel_action(op_id, node);
if (cancelled == NULL) {
crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
node);
abort_transition(INFINITY, tg_restart, "Resource op removal", match);
freeXpathObject(op_match);
free(rsc_op_xpath);
goto bail;
} else {
crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
op_id, node, cancelled->id);
}
}
freeXpathObject(op_match);
free(rsc_op_xpath);
}
bail:
freeXpathObject(xpathObj);
}
static void process_resource_updates(
const char *node, xmlNode *xml, xmlNode *change, const char *op, const char *xpath)
{
xmlNode *cIter = NULL;
xmlNode *rsc = NULL;
xmlNode *rsc_op = NULL;
int num_resources = 0;
if(xml == NULL) {
return;
} else if(strcmp((const char*)xml->name, XML_CIB_TAG_LRM) == 0) {
xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
crm_trace("Got %p in %s", xml, XML_CIB_TAG_LRM);
}
CRM_ASSERT(strcmp((const char*)xml->name, XML_LRM_TAG_RESOURCES) == 0);
for(cIter = xml->children; cIter; cIter = cIter->next) {
num_resources++;
}
if(num_resources > 1) {
/*
* Check for and fast-track the processing of LRM refreshes
* In large clusters this can result in _huge_ speedups
*
* Unfortunately we can only do so when there are no pending actions
* Otherwise we could miss updates we're waiting for and stall
*
*/
crm_debug("Detected LRM refresh - %d resources updated", num_resources);
crm_log_xml_trace(change, "lrm-refresh");
abort_transition(INFINITY, tg_restart, "LRM Refresh", NULL);
return;
}
for (rsc = __xml_first_child(xml); rsc != NULL; rsc = __xml_next(rsc)) {
crm_trace("Processing %s", ID(rsc));
for (rsc_op = __xml_first_child(rsc); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
crm_trace("Processing %s", ID(rsc_op));
process_graph_event(rsc_op, node);
}
}
}
#define NODE_PATT "/lrm[@id="
static char *get_node_from_xpath(const char *xpath)
{
char *nodeid = NULL;
char *tmp = strstr(xpath, NODE_PATT);
if(tmp) {
tmp += strlen(NODE_PATT);
tmp += 1;
nodeid = strdup(tmp);
tmp = strstr(nodeid, "\'");
CRM_ASSERT(tmp);
tmp[0] = 0;
}
return nodeid;
}
static char *extract_node_uuid(const char *xpath)
{
char *mutable_path = strdup(xpath);
char *node_uuid = NULL;
char *search = NULL;
char *match = NULL;
match = strstr(mutable_path, "node_state[@id=\'");
if (match == NULL) {
free(mutable_path);
return NULL;
}
match += strlen("node_state[@id=\'");
search = strchr(match, '\'');
if (search == NULL) {
free(mutable_path);
return NULL;
}
search[0] = 0;
node_uuid = strdup(match);
free(mutable_path);
return node_uuid;
}
static void
abort_unless_down(const char *xpath, const char *op, xmlNode *change,
const char *reason)
{
char *node_uuid = NULL;
crm_action_t *down = NULL;
if(safe_str_neq(op, "delete")) {
abort_transition(INFINITY, tg_restart, reason, change);
return;
}
node_uuid = extract_node_uuid(xpath);
if(node_uuid == NULL) {
crm_err("Could not extract node ID from %s", xpath);
abort_transition(INFINITY, tg_restart, reason, change);
return;
}
down = match_down_event(node_uuid, TRUE);
if(down == NULL || down->executed == false) {
crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
abort_transition(INFINITY, tg_restart, reason, change);
} else {
crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
}
free(node_uuid);
}
void
te_update_diff(const char *event, xmlNode * msg)
{
int rc = -EINVAL;
int format = 1;
xmlNode *change = NULL;
const char *op = NULL;
xmlNode *diff = NULL;
int p_add[] = { 0, 0, 0 };
int p_del[] = { 0, 0, 0 };
CRM_CHECK(msg != NULL, return);
crm_element_value_int(msg, F_CIB_RC, &rc);
if (transition_graph == NULL) {
crm_trace("No graph");
return;
} else if (rc < pcmk_ok) {
crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
return;
} else if (transition_graph->complete == TRUE
&& fsa_state != S_IDLE
&& fsa_state != S_TRANSITION_ENGINE && fsa_state != S_POLICY_ENGINE) {
crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
transition_graph->complete);
return;
}
op = crm_element_value(msg, F_CIB_OPERATION);
diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
xml_patch_versions(diff, p_add, p_del);
crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
fsa_state2string(fsa_state));
crm_element_value_int(diff, "format", &format);
switch(format) {
case 1:
te_legacy_update_diff(event, diff);
return;
case 2:
/* Cool, we know what to do here */
crm_log_xml_trace(diff, "Patch:Raw");
break;
default:
crm_warn("Unknown patch format: %d", format);
return;
}
for (change = __xml_first_child(diff); change != NULL; change = __xml_next(change)) {
const char *name = NULL;
const char *op = crm_element_value(change, XML_DIFF_OP);
const char *xpath = crm_element_value(change, XML_DIFF_PATH);
xmlNode *match = NULL;
const char *node = NULL;
if(op == NULL) {
continue;
} else if(strcmp(op, "create") == 0) {
match = change->children;
} else if(strcmp(op, "move") == 0) {
continue;
} else if(strcmp(op, "modify") == 0) {
match = first_named_child(change, XML_DIFF_RESULT);
if(match) {
match = match->children;
}
}
if(match) {
if (match->type == XML_COMMENT_NODE) {
crm_trace("Ignoring %s operation for comment at %s", op, xpath);
continue;
}
name = (const char *)match->name;
}
crm_trace("Handling %s operation for %s%s%s",
op, (xpath? xpath : "CIB"),
(name? " matched by " : ""), (name? name : ""));
if(xpath == NULL) {
/* Version field, ignore */
} else if(strstr(xpath, "/cib/configuration")) {
abort_transition(INFINITY, tg_restart, "Configuration change", change);
- break; /* Wont be packaged with any resource operations we may be waiting for */
+ break; /* Won't be packaged with any resource operations we may be waiting for */
} else if(strstr(xpath, "/"XML_CIB_TAG_TICKETS) || safe_str_eq(name, XML_CIB_TAG_TICKETS)) {
abort_transition(INFINITY, tg_restart, "Ticket attribute change", change);
- break; /* Wont be packaged with any resource operations we may be waiting for */
+ break; /* Won't be packaged with any resource operations we may be waiting for */
} else if(strstr(xpath, "/"XML_TAG_TRANSIENT_NODEATTRS"[") || safe_str_eq(name, XML_TAG_TRANSIENT_NODEATTRS)) {
abort_unless_down(xpath, op, change, "Transient attribute change");
- break; /* Wont be packaged with any resource operations we may be waiting for */
+ break; /* Won't be packaged with any resource operations we may be waiting for */
} else if(strstr(xpath, "/"XML_LRM_TAG_RSC_OP"[") && safe_str_eq(op, "delete")) {
crm_action_t *cancel = NULL;
char *mutable_key = strdup(xpath);
char *key, *node_uuid;
/* Extract the part of xpath between last pair of single quotes */
key = strrchr(mutable_key, '\'');
if (key != NULL) {
*key = '\0';
key = strrchr(mutable_key, '\'');
}
if (key == NULL) {
crm_warn("Ignoring malformed CIB update (resource deletion)");
free(mutable_key);
continue;
}
++key;
node_uuid = extract_node_uuid(xpath);
cancel = get_cancel_action(key, node_uuid);
if (cancel == NULL) {
abort_transition(INFINITY, tg_restart, "Resource operation removal", change);
} else {
crm_info("Cancellation of %s on %s confirmed (%d)", key, node_uuid, cancel->id);
stop_te_timer(cancel->timer);
te_action_confirmed(cancel);
update_graph(transition_graph, cancel);
trigger_graph();
}
free(mutable_key);
free(node_uuid);
} else if(strstr(xpath, "/"XML_CIB_TAG_LRM"[") && safe_str_eq(op, "delete")) {
abort_unless_down(xpath, op, change, "Resource state removal");
} else if(strstr(xpath, "/"XML_CIB_TAG_STATE"[") && safe_str_eq(op, "delete")) {
abort_unless_down(xpath, op, change, "Node state removal");
} else if(name == NULL) {
crm_debug("No result for %s operation to %s", op, xpath);
CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
} else if(strcmp(name, XML_TAG_CIB) == 0) {
xmlNode *state = NULL;
xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
xmlNode *config = first_named_child(match, XML_CIB_TAG_CONFIGURATION);
for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
node = ID(state);
process_resource_updates(node, lrm, change, op, xpath);
}
if(config) {
abort_transition(INFINITY, tg_restart, "Non-status-only change", change);
}
} else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
xmlNode *state = NULL;
for (state = __xml_first_child(match); state != NULL; state = __xml_next(state)) {
xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
node = ID(state);
process_resource_updates(node, lrm, change, op, xpath);
}
} else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
xmlNode *lrm = first_named_child(match, XML_CIB_TAG_LRM);
node = ID(match);
process_resource_updates(node, lrm, change, op, xpath);
} else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
node = ID(match);
process_resource_updates(node, match, change, op, xpath);
} else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
char *local_node = get_node_from_xpath(xpath);
process_resource_updates(local_node, match, change, op, xpath);
free(local_node);
} else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
xmlNode *rsc_op;
char *local_node = get_node_from_xpath(xpath);
for (rsc_op = __xml_first_child(match); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
process_graph_event(rsc_op, local_node);
}
free(local_node);
} else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
char *local_node = get_node_from_xpath(xpath);
process_graph_event(match, local_node);
free(local_node);
} else {
crm_err("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
}
}
}
gboolean
process_te_message(xmlNode * msg, xmlNode * xml_data)
{
const char *from = crm_element_value(msg, F_ORIG);
const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
const char *ref = crm_element_value(msg, F_CRM_REFERENCE);
const char *op = crm_element_value(msg, F_CRM_TASK);
const char *type = crm_element_value(msg, F_CRM_MSG_TYPE);
crm_trace("Processing %s (%s) message", op, ref);
crm_log_xml_trace(msg, "ipc");
if (op == NULL) {
/* error */
} else if (sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_TENGINE) != 0) {
crm_trace("Bad sys-to %s", crm_str(sys_to));
return FALSE;
} else if (safe_str_eq(op, CRM_OP_INVOKE_LRM)
&& safe_str_eq(sys_from, CRM_SYSTEM_LRMD)
/* && safe_str_eq(type, XML_ATTR_RESPONSE) */
) {
xmlXPathObject *xpathObj = NULL;
crm_log_xml_trace(msg, "Processing (N)ACK");
crm_debug("Processing (N)ACK %s from %s", crm_element_value(msg, F_CRM_REFERENCE), from);
xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
if (numXpathResults(xpathObj)) {
int lpc = 0, max = numXpathResults(xpathObj);
for (lpc = 0; lpc < max; lpc++) {
xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
const char *node = get_node_id(rsc_op);
process_graph_event(rsc_op, node);
}
freeXpathObject(xpathObj);
} else {
crm_log_xml_err(msg, "Invalid (N)ACK");
freeXpathObject(xpathObj);
return FALSE;
}
} else {
crm_err("Unknown command: %s::%s from %s", type, op, sys_from);
}
crm_trace("finished processing message");
return TRUE;
}
GHashTable *stonith_failures = NULL;
struct st_fail_rec {
int count;
int last_rc;
};
gboolean
too_many_st_failures(void)
{
GHashTableIter iter;
const char *key = NULL;
struct st_fail_rec *value = NULL;
if (stonith_failures == NULL) {
return FALSE;
}
g_hash_table_iter_init(&iter, stonith_failures);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
if (value->count > 10) {
crm_notice("Too many failures to fence %s (%d), giving up", key, value->count);
return TRUE;
} else if (value->last_rc == -ENODEV) {
crm_notice("No devices found in cluster to fence %s, giving up", key);
return TRUE;
}
}
return FALSE;
}
void
st_fail_count_reset(const char *target)
{
struct st_fail_rec *rec = NULL;
if (stonith_failures) {
rec = g_hash_table_lookup(stonith_failures, target);
}
if (rec) {
rec->count = 0;
rec->last_rc = 0;
}
}
static void
st_fail_count_increment(const char *target, int rc)
{
struct st_fail_rec *rec = NULL;
if (stonith_failures == NULL) {
stonith_failures =
g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, free);
}
rec = g_hash_table_lookup(stonith_failures, target);
if (rec) {
rec->count++;
} else {
rec = malloc(sizeof(struct st_fail_rec));
if(rec == NULL) {
return;
}
rec->count = 1;
g_hash_table_insert(stonith_failures, strdup(target), rec);
}
rec->last_rc = rc;
}
void
tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
{
char *uuid = NULL;
int target_rc = -1;
int stonith_id = -1;
int transition_id = -1;
crm_action_t *action = NULL;
int call_id = data->call_id;
int rc = data->rc;
char *userdata = data->userdata;
CRM_CHECK(userdata != NULL, return);
crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata,
pcmk_strerror(rc), rc);
if (AM_I_DC == FALSE) {
return;
}
/* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */
/* op->call_id, op->optype, op->node_name, op->op_result, */
/* (char *)op->node_list, op->private_data); */
/* filter out old STONITH actions */
CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, &target_rc),
crm_err("Invalid event detected");
goto bail;
);
if (transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid)
|| transition_graph->id != transition_id) {
crm_info("Ignoring STONITH action initiated outside of the current transition");
goto bail;
}
action = get_action(stonith_id, FALSE);
if (action == NULL) {
crm_err("Stonith action not matched");
goto bail;
}
stop_te_timer(action->timer);
if (rc == pcmk_ok) {
const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
const char *op = crm_meta_value(action->params, "stonith_action");
crm_debug("Stonith operation %d for %s passed", call_id, target);
if (action->confirmed == FALSE) {
te_action_confirmed(action);
if (action->sent_update == FALSE && safe_str_neq("on", op)) {
send_stonith_update(action, target, uuid);
action->sent_update = TRUE;
}
}
st_fail_count_reset(target);
} else {
const char *target = crm_element_value_const(action->xml, XML_LRM_ATTR_TARGET);
action->failed = TRUE;
crm_notice("Stonith operation %d for %s failed (%s): aborting transition.",
call_id, target, pcmk_strerror(rc));
abort_transition(INFINITY, tg_restart, "Stonith failed", NULL);
st_fail_count_increment(target, rc);
}
update_graph(transition_graph, action);
trigger_graph();
bail:
free(userdata);
free(uuid);
return;
}
void
cib_fencing_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
if (rc < pcmk_ok) {
crm_err("Fencing update %d for %s: failed - %s (%d)",
call_id, (char *)user_data, pcmk_strerror(rc), rc);
crm_log_xml_warn(msg, "Failed update");
abort_transition(INFINITY, tg_shutdown, "CIB update failed", NULL);
} else {
crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
}
}
void
cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
if (rc < pcmk_ok) {
crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
}
}
gboolean
action_timer_callback(gpointer data)
{
crm_action_timer_t *timer = NULL;
CRM_CHECK(data != NULL, return FALSE);
timer = (crm_action_timer_t *) data;
stop_te_timer(timer);
crm_warn("Timer popped (timeout=%d, abort_level=%d, complete=%s)",
timer->timeout,
transition_graph->abort_priority, transition_graph->complete ? "true" : "false");
CRM_CHECK(timer->action != NULL, return FALSE);
if (transition_graph->complete) {
crm_warn("Ignoring timeout while not in transition");
} else if (timer->reason == timeout_action_warn) {
print_action(LOG_WARNING, "Action missed its timeout: ", timer->action);
/* Don't check the FSA state
*
* We might also be in S_INTEGRATION or some other state waiting for this
* action so we can close the transition and continue
*/
} else {
/* fail the action */
gboolean send_update = TRUE;
const char *task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK);
print_action(LOG_ERR, "Aborting transition, action lost: ", timer->action);
timer->action->failed = TRUE;
te_action_confirmed(timer->action);
abort_transition(INFINITY, tg_restart, "Action lost", NULL);
update_graph(transition_graph, timer->action);
trigger_graph();
if (timer->action->type != action_type_rsc) {
send_update = FALSE;
} else if (safe_str_eq(task, RSC_CANCEL)) {
/* we don't need to update the CIB with these */
send_update = FALSE;
}
if (send_update) {
cib_action_update(timer->action, PCMK_LRM_OP_TIMEOUT, PCMK_OCF_UNKNOWN_ERROR);
}
}
return FALSE;
}
diff --git a/cts/CTStests.py b/cts/CTStests.py
index c4d28c3305..8aaa81e52f 100644
--- a/cts/CTStests.py
+++ b/cts/CTStests.py
@@ -1,3164 +1,3164 @@
'''CTS: Cluster Testing System: Tests module
There are a few things we want to do here:
'''
__copyright__ = '''
Copyright (C) 2000, 2001 Alan Robertson <alanr@unix.sh>
Licensed under the GNU GPL.
Add RecourceRecover testcase Zhao Kai <zhaokai@cn.ibm.com>
'''
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
#
# SPECIAL NOTE:
#
# Tests may NOT implement any cluster-manager-specific code in them.
# EXTEND the ClusterManager object to provide the base capabilities
# the test needs if you need to do something that the current CM classes
# do not. Otherwise you screw up the whole point of the object structure
# in CTS.
#
# Thank you.
#
import time, os, re, string, subprocess, tempfile
from stat import *
from cts import CTS
from cts.CTSaudits import *
from cts.CTSvars import *
from cts.patterns import PatternSelector
from cts.logging import LogFactory
from cts.remote import RemoteFactory
from cts.watcher import LogWatcher
from cts.environment import EnvFactory
AllTestClasses = [ ]
class CTSTest:
'''
A Cluster test.
We implement the basic set of properties and behaviors for a generic
cluster test.
Cluster tests track their own statistics.
We keep each of the kinds of counts we track as separate {name,value}
pairs.
'''
def __init__(self, cm):
#self.name="the unnamed test"
self.Stats = {"calls":0
, "success":0
, "failure":0
, "skipped":0
, "auditfail":0}
# if not issubclass(cm.__class__, ClusterManager):
# raise ValueError("Must be a ClusterManager object")
self.CM = cm
self.Env = EnvFactory().getInstance()
self.rsh = RemoteFactory().getInstance()
self.logger = LogFactory()
self.templates = PatternSelector(cm["Name"])
self.Audits = []
self.timeout = 120
self.passed = 1
self.is_loop = 0
self.is_unsafe = 0
self.is_docker_unsafe = 0
self.is_experimental = 0
self.is_container = 0
self.is_valgrind = 0
self.benchmark = 0 # which tests to benchmark
self.timer = {} # timers
def log(self, args):
self.logger.log(args)
def debug(self, args):
self.logger.debug(args)
def has_key(self, key):
return key in self.Stats
def __setitem__(self, key, value):
self.Stats[key] = value
def __getitem__(self, key):
if str(key) == "0":
raise ValueError("Bad call to 'foo in X', should reference 'foo in X.Stats' instead")
if key in self.Stats:
return self.Stats[key]
return None
def log_mark(self, msg):
self.debug("MARK: test %s %s %d" % (self.name,msg,time.time()))
return
def get_timer(self,key = "test"):
try: return self.timer[key]
except: return 0
def set_timer(self,key = "test"):
self.timer[key] = time.time()
return self.timer[key]
def log_timer(self,key = "test"):
elapsed = 0
if key in self.timer:
elapsed = time.time() - self.timer[key]
s = key == "test" and self.name or "%s:%s" % (self.name,key)
self.debug("%s runtime: %.2f" % (s, elapsed))
del self.timer[key]
return elapsed
def incr(self, name):
'''Increment (or initialize) the value associated with the given name'''
if not name in self.Stats:
self.Stats[name] = 0
self.Stats[name] = self.Stats[name]+1
# Reset the test passed boolean
if name == "calls":
self.passed = 1
def failure(self, reason="none"):
'''Increment the failure count'''
self.passed = 0
self.incr("failure")
self.logger.log(("Test %s" % self.name).ljust(35) + " FAILED: %s" % reason)
return None
def success(self):
'''Increment the success count'''
self.incr("success")
return 1
def skipped(self):
'''Increment the skipped count'''
self.incr("skipped")
return 1
def __call__(self, node):
'''Perform the given test'''
raise ValueError("Abstract Class member (__call__)")
self.incr("calls")
return self.failure()
def audit(self):
passed = 1
if len(self.Audits) > 0:
for audit in self.Audits:
if not audit():
self.logger.log("Internal %s Audit %s FAILED." % (self.name, audit.name()))
self.incr("auditfail")
passed = 0
return passed
def setup(self, node):
'''Setup the given test'''
return self.success()
def teardown(self, node):
'''Tear down the given test'''
return self.success()
def create_watch(self, patterns, timeout, name=None):
if not name:
name = self.name
return LogWatcher(self.Env["LogFileName"], patterns, name, timeout, kind=self.Env["LogWatcher"], hosts=self.Env["nodes"])
def local_badnews(self, prefix, watch, local_ignore=[]):
errcount = 0
if not prefix:
prefix = "LocalBadNews:"
ignorelist = []
ignorelist.append(" CTS: ")
ignorelist.append(prefix)
ignorelist.extend(local_ignore)
while errcount < 100:
match = watch.look(0)
if match:
add_err = 1
for ignore in ignorelist:
if add_err == 1 and re.search(ignore, match):
add_err = 0
if add_err == 1:
self.logger.log(prefix + " " + match)
errcount = errcount + 1
else:
break
else:
self.logger.log("Too many errors!")
watch.end()
return errcount
def is_applicable(self):
return self.is_applicable_common()
def is_applicable_common(self):
'''Return TRUE if we are applicable in the current test configuration'''
#raise ValueError("Abstract Class member (is_applicable)")
if self.is_loop and not self.Env["loop-tests"]:
return 0
elif self.is_unsafe and not self.Env["unsafe-tests"]:
return 0
elif self.is_valgrind and not self.Env["valgrind-tests"]:
return 0
elif self.is_experimental and not self.Env["experimental-tests"]:
return 0
elif self.is_docker_unsafe and self.Env["docker"]:
return 0
elif self.is_container and not self.Env["container-tests"]:
return 0
elif self.Env["benchmark"] and self.benchmark == 0:
return 0
return 1
def find_ocfs2_resources(self, node):
self.r_o2cb = None
self.r_ocfs2 = []
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rtype == "o2cb" and r.parent != "NA":
self.debug("Found o2cb: %s" % self.r_o2cb)
self.r_o2cb = r.parent
if re.search("^Constraint", line):
c = AuditConstraint(self.CM, line)
if c.type == "rsc_colocation" and c.target == self.r_o2cb:
self.r_ocfs2.append(c.rsc)
self.debug("Found ocfs2 filesystems: %s" % repr(self.r_ocfs2))
return len(self.r_ocfs2)
def canrunnow(self, node):
'''Return TRUE if we can meaningfully run right now'''
return 1
def errorstoignore(self):
'''Return list of errors which are 'normal' and should be ignored'''
return []
class StopTest(CTSTest):
'''Stop (deactivate) the cluster manager on a node'''
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "Stop"
def __call__(self, node):
'''Perform the 'stop' test. '''
self.incr("calls")
if self.CM.ShouldBeStatus[node] != "up":
return self.skipped()
patterns = []
# Technically we should always be able to notice ourselves stopping
patterns.append(self.templates["Pat:We_stopped"] % node)
#if self.Env["use_logd"]:
# patterns.append(self.templates["Pat:Logd_stopped"] % node)
# Any active node needs to notice this one left
# NOTE: This wont work if we have multiple partitions
for other in self.Env["nodes"]:
if self.CM.ShouldBeStatus[other] == "up" and other != node:
patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
#self.debug("Checking %s will notice %s left"%(other, node))
watch = self.create_watch(patterns, self.Env["DeadTime"])
watch.setwatch()
if node == self.CM.OurNode:
self.incr("us")
else:
if self.CM.upcount() <= 1:
self.incr("all")
else:
self.incr("them")
self.CM.StopaCM(node)
watch_result = watch.lookforall()
failreason = None
UnmatchedList = "||"
if watch.unmatched:
(rc, output) = self.rsh(node, "/bin/ps axf", None)
for line in output:
self.debug(line)
(rc, output) = self.rsh(node, "/usr/sbin/dlm_tool dump", None)
for line in output:
self.debug(line)
for regex in watch.unmatched:
self.logger.log ("ERROR: Shutdown pattern not found: %s" % (regex))
UnmatchedList += regex + "||";
failreason = "Missing shutdown pattern"
self.CM.cluster_stable(self.Env["DeadTime"])
if not watch.unmatched or self.CM.upcount() == 0:
return self.success()
if len(watch.unmatched) >= self.CM.upcount():
return self.failure("no match against (%s)" % UnmatchedList)
if failreason == None:
return self.success()
else:
return self.failure(failreason)
#
# We don't register StopTest because it's better when called by
# another test...
#
class StartTest(CTSTest):
'''Start (activate) the cluster manager on a node'''
def __init__(self, cm, debug=None):
CTSTest.__init__(self,cm)
self.name = "start"
self.debug = debug
def __call__(self, node):
'''Perform the 'start' test. '''
self.incr("calls")
if self.CM.upcount() == 0:
self.incr("us")
else:
self.incr("them")
if self.CM.ShouldBeStatus[node] != "down":
return self.skipped()
elif self.CM.StartaCM(node):
return self.success()
else:
return self.failure("Startup %s on node %s failed"
% (self.Env["Name"], node))
#
# We don't register StartTest because it's better when called by
# another test...
#
class FlipTest(CTSTest):
'''If it's running, stop it. If it's stopped start it.
Overthrow the status quo...
'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Flip"
self.start = StartTest(cm)
self.stop = StopTest(cm)
def __call__(self, node):
'''Perform the 'Flip' test. '''
self.incr("calls")
if self.CM.ShouldBeStatus[node] == "up":
self.incr("stopped")
ret = self.stop(node)
type = "up->down"
# Give the cluster time to recognize it's gone...
time.sleep(self.Env["StableTime"])
elif self.CM.ShouldBeStatus[node] == "down":
self.incr("started")
ret = self.start(node)
type = "down->up"
else:
return self.skipped()
self.incr(type)
if ret:
return self.success()
else:
return self.failure("%s failure" % type)
# Register FlipTest as a good test to run
AllTestClasses.append(FlipTest)
class RestartTest(CTSTest):
'''Stop and restart a node'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Restart"
self.start = StartTest(cm)
self.stop = StopTest(cm)
self.benchmark = 1
def __call__(self, node):
'''Perform the 'restart' test. '''
self.incr("calls")
self.incr("node:" + node)
ret1 = 1
if self.CM.StataCM(node):
self.incr("WasStopped")
if not self.start(node):
return self.failure("start (setup) failure: "+node)
self.set_timer()
if not self.stop(node):
return self.failure("stop failure: "+node)
if not self.start(node):
return self.failure("start failure: "+node)
return self.success()
# Register RestartTest as a good test to run
AllTestClasses.append(RestartTest)
class StonithdTest(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "Stonithd"
self.startall = SimulStartLite(cm)
self.benchmark = 1
def __call__(self, node):
self.incr("calls")
if len(self.Env["nodes"]) < 2:
return self.skipped()
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
is_dc = self.CM.is_node_dc(node)
watchpats = []
watchpats.append(self.templates["Pat:FenceOpOK"] % node)
watchpats.append(self.templates["Pat:NodeFenced"] % node)
if self.Env["at-boot"] == 0:
self.debug("Expecting %s to stay down" % node)
self.CM.ShouldBeStatus[node] = "down"
else:
self.debug("Expecting %s to come up again %d" % (node, self.Env["at-boot"]))
watchpats.append("%s.* S_STARTING -> S_PENDING" % node)
watchpats.append("%s.* S_PENDING -> S_NOT_DC" % node)
watch = self.create_watch(watchpats, 30 + self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
watch.setwatch()
origin = self.Env.RandomGen.choice(self.Env["nodes"])
rc = self.rsh(origin, "stonith_admin --reboot %s -VVVVVV" % node)
if rc == 194:
# 194 - 256 = -62 = Timer expired
#
# Look for the patterns, usually this means the required
# device was running on the node to be fenced - or that
# the required devices were in the process of being loaded
# and/or moved
#
# Effectively the node committed suicide so there will be
# no confirmation, but pacemaker should be watching and
# fence the node again
self.logger.log("Fencing command on %s to fence %s timed out" % (origin, node))
elif origin != node and rc != 0:
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
self.debug("Waiting STONITHd node to come back up")
self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)
self.logger.log("Fencing command on %s failed to fence %s (rc=%d)" % (origin, node, rc))
elif origin == node and rc != 255:
# 255 == broken pipe, ie. the node was fenced as expected
self.logger.log("Locally originated fencing returned %d" % rc)
self.set_timer("fence")
matched = watch.lookforall()
self.log_timer("fence")
self.set_timer("reform")
if watch.unmatched:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
self.debug("Waiting STONITHd node to come back up")
self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)
self.debug("Waiting for the cluster to re-stabilize with all nodes")
is_stable = self.CM.cluster_stable(self.Env["StartTime"])
if not matched:
return self.failure("Didn't find all expected patterns")
elif not is_stable:
return self.failure("Cluster did not become stable")
self.log_timer("reform")
return self.success()
def errorstoignore(self):
return [
self.templates["Pat:Fencing_start"] % ".*",
self.templates["Pat:Fencing_ok"] % ".*",
r"error.*: Resource .*stonith::.* is active on 2 nodes attempting recovery",
r"error.*: Operation reboot of .*by .* for stonith_admin.*: Timer expired",
]
def is_applicable(self):
if not self.is_applicable_common():
return 0
if "DoFencing" in self.Env.keys():
return self.Env["DoFencing"]
return 1
AllTestClasses.append(StonithdTest)
class StartOnebyOne(CTSTest):
'''Start all the nodes ~ one by one'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "StartOnebyOne"
self.stopall = SimulStopLite(cm)
self.start = StartTest(cm)
self.ns = CTS.NodeStatus(cm.Env)
def __call__(self, dummy):
'''Perform the 'StartOnebyOne' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Shut down all the nodes...
ret = self.stopall(None)
if not ret:
return self.failure("Test setup failed")
failed = []
self.set_timer()
for node in self.Env["nodes"]:
if not self.start(node):
failed.append(node)
if len(failed) > 0:
return self.failure("Some node failed to start: " + repr(failed))
return self.success()
# Register StartOnebyOne as a good test to run
AllTestClasses.append(StartOnebyOne)
class SimulStart(CTSTest):
'''Start all the nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStart"
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
def __call__(self, dummy):
'''Perform the 'SimulStart' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Shut down all the nodes...
ret = self.stopall(None)
if not ret:
return self.failure("Setup failed")
self.CM.clear_all_caches()
if not self.startall(None):
return self.failure("Startall failed")
return self.success()
# Register SimulStart as a good test to run
AllTestClasses.append(SimulStart)
class SimulStop(CTSTest):
'''Stop all the nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStop"
self.startall = SimulStartLite(cm)
self.stopall = SimulStopLite(cm)
def __call__(self, dummy):
'''Perform the 'SimulStop' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Start up all the nodes...
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
if not self.stopall(None):
return self.failure("Stopall failed")
return self.success()
# Register SimulStop as a good test to run
AllTestClasses.append(SimulStop)
class StopOnebyOne(CTSTest):
'''Stop all the nodes in order'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "StopOnebyOne"
self.startall = SimulStartLite(cm)
self.stop = StopTest(cm)
def __call__(self, dummy):
'''Perform the 'StopOnebyOne' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Start up all the nodes...
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
failed = []
self.set_timer()
for node in self.Env["nodes"]:
if not self.stop(node):
failed.append(node)
if len(failed) > 0:
return self.failure("Some node failed to stop: " + repr(failed))
self.CM.clear_all_caches()
return self.success()
# Register StopOnebyOne as a good test to run
AllTestClasses.append(StopOnebyOne)
class RestartOnebyOne(CTSTest):
'''Restart all the nodes in order'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "RestartOnebyOne"
self.startall = SimulStartLite(cm)
def __call__(self, dummy):
'''Perform the 'RestartOnebyOne' test. '''
self.incr("calls")
# We ignore the "node" parameter...
# Start up all the nodes...
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
did_fail = []
self.set_timer()
self.restart = RestartTest(self.CM)
for node in self.Env["nodes"]:
if not self.restart(node):
did_fail.append(node)
if did_fail:
return self.failure("Could not restart %d nodes: %s"
% (len(did_fail), repr(did_fail)))
return self.success()
# Register StopOnebyOne as a good test to run
AllTestClasses.append(RestartOnebyOne)
class PartialStart(CTSTest):
'''Start a node - but tell it to stop before it finishes starting up'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "PartialStart"
self.startall = SimulStartLite(cm)
self.stopall = SimulStopLite(cm)
self.stop = StopTest(cm)
#self.is_unsafe = 1
def __call__(self, node):
'''Perform the 'PartialStart' test. '''
self.incr("calls")
ret = self.stopall(None)
if not ret:
return self.failure("Setup failed")
# FIXME! This should use the CM class to get the pattern
# then it would be applicable in general
watchpats = []
watchpats.append("crmd.*Connecting to cluster infrastructure")
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
self.CM.StartaCMnoBlock(node)
ret = watch.lookforall()
if not ret:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
return self.failure("Setup of %s failed" % node)
ret = self.stop(node)
if not ret:
return self.failure("%s did not stop in time" % node)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
# We might do some fencing in the 2-node case if we make it up far enough
return [
r"Executing reboot fencing operation",
r"Requesting fencing \([^)]+\) of node ",
]
# Register StopOnebyOne as a good test to run
AllTestClasses.append(PartialStart)
class StandbyTest(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Standby"
self.benchmark = 1
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
# make sure the node is active
# set the node to standby mode
# check resources, none resource should be running on the node
# set the node to active mode
# check resouces, resources should have been migrated back (SHOULD THEY?)
def __call__(self, node):
self.incr("calls")
ret = self.startall(None)
if not ret:
return self.failure("Start all nodes failed")
self.debug("Make sure node %s is active" % node)
if self.CM.StandbyStatus(node) != "off":
if not self.CM.SetStandbyMode(node, "off"):
return self.failure("can't set node %s to active mode" % node)
self.CM.cluster_stable()
status = self.CM.StandbyStatus(node)
if status != "off":
return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
self.debug("Getting resources running on node %s" % node)
rsc_on_node = self.CM.active_resources(node)
watchpats = []
watchpats.append(r"State transition .* -> S_POLICY_ENGINE")
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
self.debug("Setting node %s to standby mode" % node)
if not self.CM.SetStandbyMode(node, "on"):
return self.failure("can't set node %s to standby mode" % node)
self.set_timer("on")
ret = watch.lookforall()
if not ret:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
self.CM.SetStandbyMode(node, "off")
return self.failure("cluster didn't react to standby change on %s" % node)
self.CM.cluster_stable()
status = self.CM.StandbyStatus(node)
if status != "on":
return self.failure("standby status of %s is [%s] but we expect [on]" % (node, status))
self.log_timer("on")
self.debug("Checking resources")
bad_run = self.CM.active_resources(node)
if len(bad_run) > 0:
rc = self.failure("%s set to standby, %s is still running on it" % (node, repr(bad_run)))
self.debug("Setting node %s to active mode" % node)
self.CM.SetStandbyMode(node, "off")
return rc
self.debug("Setting node %s to active mode" % node)
if not self.CM.SetStandbyMode(node, "off"):
return self.failure("can't set node %s to active mode" % node)
self.set_timer("off")
self.CM.cluster_stable()
status = self.CM.StandbyStatus(node)
if status != "off":
return self.failure("standby status of %s is [%s] but we expect [off]" % (node, status))
self.log_timer("off")
return self.success()
AllTestClasses.append(StandbyTest)
class ValgrindTest(CTSTest):
'''Check for memory leaks'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Valgrind"
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
self.is_valgrind = 1
self.is_loop = 1
def setup(self, node):
self.incr("calls")
ret = self.stopall(None)
if not ret:
return self.failure("Stop all nodes failed")
# Enable valgrind
self.logger.logPat = "/tmp/%s-*.valgrind" % self.name
self.Env["valgrind-prefix"] = self.name
self.rsh(node, "rm -f %s" % self.logger.logPat, None)
ret = self.startall(None)
if not ret:
return self.failure("Start all nodes failed")
for node in self.Env["nodes"]:
(rc, output) = self.rsh(node, "ps u --ppid `pidofproc aisexec`", None)
for line in output:
self.debug(line)
return self.success()
def teardown(self, node):
# Disable valgrind
self.Env["valgrind-prefix"] = None
# Return all nodes to normal
ret = self.stopall(None)
if not ret:
return self.failure("Stop all nodes failed")
return self.success()
def find_leaks(self):
# Check for leaks
leaked = []
self.stop = StopTest(self.CM)
for node in self.Env["nodes"]:
(rc, ps_out) = self.rsh(node, "ps u --ppid `pidofproc aisexec`", None)
rc = self.stop(node)
if not rc:
self.failure("Couldn't shut down %s" % node)
rc = self.rsh(node, "grep -e indirectly.*lost:.*[1-9] -e definitely.*lost:.*[1-9] -e (ERROR|error).*SUMMARY:.*[1-9].*errors %s" % self.logger.logPat, 0)
if rc != 1:
leaked.append(node)
self.failure("Valgrind errors detected on %s" % node)
for line in ps_out:
self.logger.log(line)
(rc, output) = self.rsh(node, "grep -e lost: -e SUMMARY: %s" % self.logger.logPat, None)
for line in output:
self.logger.log(line)
(rc, output) = self.rsh(node, "cat %s" % self.logger.logPat, None)
for line in output:
self.debug(line)
self.rsh(node, "rm -f %s" % self.logger.logPat, None)
return leaked
def __call__(self, node):
leaked = self.find_leaks()
if len(leaked) > 0:
return self.failure("Nodes %s leaked" % repr(leaked))
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"cib.*: \*\*\*\*\*\*\*\*\*\*\*\*\*",
r"cib.*: .* avoid confusing Valgrind",
r"HA_VALGRIND_ENABLED",
]
class StandbyLoopTest(ValgrindTest):
'''Check for memory leaks by putting a node in and out of standby for an hour'''
def __init__(self, cm):
ValgrindTest.__init__(self,cm)
self.name = "StandbyLoop"
def __call__(self, node):
lpc = 0
delay = 2
failed = 0
done = time.time() + self.Env["loop-minutes"] * 60
while time.time() <= done and not failed:
lpc = lpc + 1
time.sleep(delay)
if not self.CM.SetStandbyMode(node, "on"):
self.failure("can't set node %s to standby mode" % node)
failed = lpc
time.sleep(delay)
if not self.CM.SetStandbyMode(node, "off"):
self.failure("can't set node %s to active mode" % node)
failed = lpc
leaked = self.find_leaks()
if failed:
return self.failure("Iteration %d failed" % failed)
elif len(leaked) > 0:
return self.failure("Nodes %s leaked" % repr(leaked))
return self.success()
AllTestClasses.append(StandbyLoopTest)
class BandwidthTest(CTSTest):
# Tests should not be cluster-manager-specific
# If you need to find out cluster manager configuration to do this, then
# it should be added to the generic cluster manager API.
'''Test the bandwidth which heartbeat uses'''
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "Bandwidth"
self.start = StartTest(cm)
self.__setitem__("min",0)
self.__setitem__("max",0)
self.__setitem__("totalbandwidth",0)
(handle, self.tempfile) = tempfile.mkstemp(".cts")
os.close(handle)
self.startall = SimulStartLite(cm)
def __call__(self, node):
'''Perform the Bandwidth test'''
self.incr("calls")
if self.CM.upcount() < 1:
return self.skipped()
Path = self.CM.InternalCommConfig()
if "ip" not in Path["mediatype"]:
return self.skipped()
port = Path["port"][0]
port = int(port)
ret = self.startall(None)
if not ret:
return self.failure("Test setup failed")
time.sleep(5) # We get extra messages right after startup.
fstmpfile = "/var/run/band_estimate"
dumpcmd = "tcpdump -p -n -c 102 -i any udp port %d > %s 2>&1" \
% (port, fstmpfile)
rc = self.rsh(node, dumpcmd)
if rc == 0:
farfile = "root@%s:%s" % (node, fstmpfile)
self.rsh.cp(farfile, self.tempfile)
Bandwidth = self.countbandwidth(self.tempfile)
if not Bandwidth:
self.logger.log("Could not compute bandwidth.")
return self.success()
intband = int(Bandwidth + 0.5)
self.logger.log("...bandwidth: %d bits/sec" % intband)
self.Stats["totalbandwidth"] = self.Stats["totalbandwidth"] + Bandwidth
if self.Stats["min"] == 0:
self.Stats["min"] = Bandwidth
if Bandwidth > self.Stats["max"]:
self.Stats["max"] = Bandwidth
if Bandwidth < self.Stats["min"]:
self.Stats["min"] = Bandwidth
self.rsh(node, "rm -f %s" % fstmpfile)
os.unlink(self.tempfile)
return self.success()
else:
return self.failure("no response from tcpdump command [%d]!" % rc)
def countbandwidth(self, file):
fp = open(file, "r")
fp.seek(0)
count = 0
sum = 0
while 1:
line = fp.readline()
if not line:
return None
if re.search("udp",line) or re.search("UDP,", line):
count = count + 1
linesplit = string.split(line," ")
for j in range(len(linesplit)-1):
if linesplit[j] == "udp": break
if linesplit[j] == "length:": break
try:
sum = sum + int(linesplit[j+1])
except ValueError:
self.logger.log("Invalid tcpdump line: %s" % line)
return None
T1 = linesplit[0]
timesplit = string.split(T1,":")
time2split = string.split(timesplit[2],".")
time1 = (int(timesplit[0])*60+int(timesplit[1]))*60+int(time2split[0])+int(time2split[1])*0.000001
break
while count < 100:
line = fp.readline()
if not line:
return None
if re.search("udp",line) or re.search("UDP,", line):
count = count+1
linessplit = string.split(line," ")
for j in range(len(linessplit)-1):
if linessplit[j] == "udp": break
if linesplit[j] == "length:": break
try:
sum = int(linessplit[j+1]) + sum
except ValueError:
self.logger.log("Invalid tcpdump line: %s" % line)
return None
T2 = linessplit[0]
timesplit = string.split(T2,":")
time2split = string.split(timesplit[2],".")
time2 = (int(timesplit[0])*60+int(timesplit[1]))*60+int(time2split[0])+int(time2split[1])*0.000001
time = time2-time1
if (time <= 0):
return 0
return (sum*8)/time
def is_applicable(self):
'''BandwidthTest never applicable'''
return 0
AllTestClasses.append(BandwidthTest)
###################################################################
class MaintenanceMode(CTSTest):
###################################################################
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "MaintenanceMode"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.max = 30
#self.is_unsafe = 1
self.benchmark = 1
self.action = "asyncmon"
self.interval = 0
self.rid = "maintenanceDummy"
def toggleMaintenanceMode(self, node, action):
pats = []
pats.append(self.templates["Pat:DC_IDLE"])
# fail the resource right after turning Maintenance mode on
# verify it is not recovered until maintenance mode is turned off
if action == "On":
pats.append(r"pengine.*:\s+warning:.*Processing failed op %s for %s on" % (self.action, self.rid))
else:
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.debug("Turning maintenance mode %s" % action)
self.rsh(node, self.templates["MaintenanceMode%s" % (action)])
if (action == "On"):
self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))
self.set_timer("recover%s" % (action))
watch.lookforall()
self.log_timer("recover%s" % (action))
if watch.unmatched:
self.debug("Failed to find patterns when turning maintenance mode %s" % action)
return repr(watch.unmatched)
return ""
def insertMaintenanceDummy(self, node):
pats = []
pats.append(("%s.*" % node) + (self.templates["Pat:RscOpOK"] % ("start", self.rid)))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.CM.AddDummyRsc(node, self.rid)
self.set_timer("addDummy")
watch.lookforall()
self.log_timer("addDummy")
if watch.unmatched:
self.debug("Failed to find patterns when adding maintenance dummy resource")
return repr(watch.unmatched)
return ""
def removeMaintenanceDummy(self, node):
pats = []
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.CM.RemoveDummyRsc(node, self.rid)
self.set_timer("removeDummy")
watch.lookforall()
self.log_timer("removeDummy")
if watch.unmatched:
self.debug("Failed to find patterns when removing maintenance dummy resource")
return repr(watch.unmatched)
return ""
def managedRscList(self, node):
rscList = []
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
tmp = AuditResource(self.CM, line)
if tmp.managed():
rscList.append(tmp.id)
return rscList
def verifyResources(self, node, rscList, managed):
managedList = list(rscList)
managed_str = "managed"
if not managed:
managed_str = "unmanaged"
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
tmp = AuditResource(self.CM, line)
if managed and not tmp.managed():
continue
elif not managed and tmp.managed():
continue
elif managedList.count(tmp.id):
managedList.remove(tmp.id)
if len(managedList) == 0:
self.debug("Found all %s resources on %s" % (managed_str, node))
return True
self.logger.log("Could not find all %s resources on %s. %s" % (managed_str, node, managedList))
return False
def __call__(self, node):
'''Perform the 'MaintenanceMode' test. '''
self.incr("calls")
verify_managed = False
verify_unmanaged = False
failPat = ""
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
# get a list of all the managed resources. We use this list
# after enabling maintenance mode to verify all managed resources
# become un-managed. After maintenance mode is turned off, we use
# this list to verify all the resources become managed again.
managedResources = self.managedRscList(node)
if len(managedResources) == 0:
self.logger.log("No managed resources on %s" % node)
return self.skipped()
# insert a fake resource we can fail during maintenance mode
# so we can verify recovery does not take place until after maintenance
# mode is disabled.
failPat = failPat + self.insertMaintenanceDummy(node)
# toggle maintenance mode ON, then fail dummy resource.
failPat = failPat + self.toggleMaintenanceMode(node, "On")
# verify all the resources are now unmanaged
if self.verifyResources(node, managedResources, False):
verify_unmanaged = True
# Toggle maintenance mode OFF, verify dummy is recovered.
failPat = failPat + self.toggleMaintenanceMode(node, "Off")
# verify all the resources are now managed again
if self.verifyResources(node, managedResources, True):
verify_managed = True
# Remove our maintenance dummy resource.
failPat = failPat + self.removeMaintenanceDummy(node)
self.CM.cluster_stable()
if failPat != "":
return self.failure("Unmatched patterns: %s" % (failPat))
elif verify_unmanaged is False:
return self.failure("Failed to verify resources became unmanaged during maintenance mode")
elif verify_managed is False:
return self.failure("Failed to verify resources switched back to managed after disabling maintenance mode")
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"Updating failcount for %s" % self.rid,
r"pengine.*: Recover %s\s*\(.*\)" % self.rid,
r"Unknown operation: fail",
r"(ERROR|error): sending stonithRA op to stonithd failed.",
self.templates["Pat:RscOpOK"] % (self.action, self.rid),
r"(ERROR|error).*: Action %s_%s_%d .* initiated outside of a transition" % (self.rid, self.action, self.interval),
]
AllTestClasses.append(MaintenanceMode)
class ResourceRecover(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "ResourceRecover"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.max = 30
self.rid = None
self.rid_alt = None
#self.is_unsafe = 1
self.benchmark = 1
# these are the values used for the new LRM API call
self.action = "asyncmon"
self.interval = 0
def __call__(self, node):
'''Perform the 'ResourceRecover' test. '''
self.incr("calls")
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
resourcelist = self.CM.active_resources(node)
# if there are no resourcelist, return directly
if len(resourcelist) == 0:
self.logger.log("No active resources on %s" % node)
return self.skipped()
self.rid = self.Env.RandomGen.choice(resourcelist)
self.rid_alt = self.rid
rsc = None
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
tmp = AuditResource(self.CM, line)
if tmp.id == self.rid:
rsc = tmp
# Handle anonymous clones that get renamed
self.rid = rsc.clone_id
break
if not rsc:
return self.failure("Could not find %s in the resource list" % self.rid)
self.debug("Shooting %s aka. %s" % (rsc.clone_id, rsc.id))
pats = []
pats.append(r"pengine.*:\s+warning:.*Processing failed op %s for (%s|%s) on" % (self.action,
rsc.id, rsc.clone_id))
if rsc.managed():
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.rid))
if rsc.unique():
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.rid))
else:
# Anonymous clones may get restarted with a different clone number
pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
watch = self.create_watch(pats, 60)
watch.setwatch()
self.rsh(node, "crm_resource -V -F -r %s -H %s &>/dev/null" % (self.rid, node))
self.set_timer("recover")
watch.lookforall()
self.log_timer("recover")
self.CM.cluster_stable()
recovered = self.CM.ResourceLocation(self.rid)
if watch.unmatched:
return self.failure("Patterns not found: %s" % repr(watch.unmatched))
elif rsc.unique() and len(recovered) > 1:
return self.failure("%s is now active on more than one node: %s"%(self.rid, repr(recovered)))
elif len(recovered) > 0:
self.debug("%s is running on: %s" % (self.rid, repr(recovered)))
elif rsc.managed():
return self.failure("%s was not recovered and is inactive" % self.rid)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"Updating failcount for %s" % self.rid,
r"pengine.*: Recover (%s|%s)\s*\(.*\)" % (self.rid, self.rid_alt),
r"Unknown operation: fail",
r"(ERROR|error): sending stonithRA op to stonithd failed.",
self.templates["Pat:RscOpOK"] % (self.action, self.rid),
r"(ERROR|error).*: Action %s_%s_%d .* initiated outside of a transition" % (self.rid, self.action, self.interval),
]
AllTestClasses.append(ResourceRecover)
class ComponentFail(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "ComponentFail"
# TODO make this work correctly in docker.
self.is_docker_unsafe = 1
self.startall = SimulStartLite(cm)
self.complist = cm.Components()
self.patterns = []
self.okerrpatterns = []
self.is_unsafe = 1
def __call__(self, node):
'''Perform the 'ComponentFail' test. '''
self.incr("calls")
self.patterns = []
self.okerrpatterns = []
# start all nodes
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
if not self.CM.cluster_stable(self.Env["StableTime"]):
return self.failure("Setup failed - unstable")
node_is_dc = self.CM.is_node_dc(node, None)
# select a component to kill
chosen = self.Env.RandomGen.choice(self.complist)
while chosen.dc_only == 1 and node_is_dc == 0:
chosen = self.Env.RandomGen.choice(self.complist)
self.debug("...component %s (dc=%d,boot=%d)" % (chosen.name, node_is_dc,chosen.triggersreboot))
self.incr(chosen.name)
if chosen.name != "aisexec" and chosen.name != "corosync":
if self.Env["Name"] != "crm-lha" or chosen.name != "pengine":
self.patterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
self.patterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))
self.patterns.extend(chosen.pats)
if node_is_dc:
self.patterns.extend(chosen.dc_pats)
# In an ideal world, this next stuff should be in the "chosen" object as a member function
if self.Env["Name"] == "crm-lha" and chosen.triggersreboot:
# Make sure the node goes down and then comes back up if it should reboot...
for other in self.Env["nodes"]:
if other != node:
self.patterns.append(self.templates["Pat:They_stopped"] %(other, self.CM.key_for_node(node)))
self.patterns.append(self.templates["Pat:Slave_started"] % node)
self.patterns.append(self.templates["Pat:Local_started"] % node)
if chosen.dc_only:
# Sometimes these will be in the log, and sometimes they won't...
self.okerrpatterns.append("%s .*Process %s:.* exited" % (node, chosen.name))
self.okerrpatterns.append("%s .*I_ERROR.*crmdManagedChildDied" % node)
self.okerrpatterns.append("%s .*The %s subsystem terminated unexpectedly" % (node, chosen.name))
self.okerrpatterns.append("(ERROR|error): Client .* exited with return code")
else:
# Sometimes this won't be in the log...
self.okerrpatterns.append(self.templates["Pat:ChildKilled"] %(node, chosen.name))
self.okerrpatterns.append(self.templates["Pat:ChildRespawn"] %(node, chosen.name))
self.okerrpatterns.append(self.templates["Pat:ChildExit"])
if chosen.name == "stonith":
# Ignore actions for STONITH resources
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rclass == "stonith":
self.okerrpatterns.append(self.templates["Pat:Fencing_recover"] % r.id)
# supply a copy so self.patterns doesn't end up empty
tmpPats = []
tmpPats.extend(self.patterns)
self.patterns.extend(chosen.badnews_ignore)
# Look for STONITH ops, depending on Env["at-boot"] we might need to change the nodes status
stonithPats = []
stonithPats.append(self.templates["Pat:Fencing_ok"] % node)
stonith = self.create_watch(stonithPats, 0)
stonith.setwatch()
# set the watch for stable
watch = self.create_watch(
tmpPats, self.Env["DeadTime"] + self.Env["StableTime"] + self.Env["StartTime"])
watch.setwatch()
# kill the component
chosen.kill(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
self.debug("Waiting for any STONITHd node to come back up")
self.CM.ns.WaitForAllNodesToComeUp(self.Env["nodes"], 600)
self.debug("Waiting for the cluster to re-stabilize with all nodes")
self.CM.cluster_stable(self.Env["StartTime"])
self.debug("Checking if %s was shot" % node)
shot = stonith.look(60)
if shot:
self.debug("Found: " + repr(shot))
self.okerrpatterns.append(self.templates["Pat:Fencing_start"] % node)
if self.Env["at-boot"] == 0:
self.CM.ShouldBeStatus[node] = "down"
# If fencing occurred, chances are many (if not all) the expected logs
# will not be sent - or will be lost when the node reboots
return self.success()
# check for logs indicating a graceful recovery
matched = watch.lookforall(allow_multiple_matches=1)
if watch.unmatched:
self.logger.log("Patterns not found: " + repr(watch.unmatched))
self.debug("Waiting for the cluster to re-stabilize with all nodes")
is_stable = self.CM.cluster_stable(self.Env["StartTime"])
if not matched:
return self.failure("Didn't find all expected %s patterns" % chosen.name)
elif not is_stable:
return self.failure("Cluster did not become stable after killing %s" % chosen.name)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
# Note that okerrpatterns refers to the last time we ran this test
# The good news is that this works fine for us...
self.okerrpatterns.extend(self.patterns)
return self.okerrpatterns
AllTestClasses.append(ComponentFail)
class SplitBrainTest(CTSTest):
'''It is used to test split-brain. when the path between the two nodes break
check the two nodes both take over the resource'''
def __init__(self,cm):
CTSTest.__init__(self,cm)
self.name = "SplitBrain"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.is_experimental = 1
def isolate_partition(self, partition):
other_nodes = []
other_nodes.extend(self.Env["nodes"])
for node in partition:
try:
other_nodes.remove(node)
except ValueError:
self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]) + " from " +repr(partition))
if len(other_nodes) == 0:
return 1
self.debug("Creating partition: " + repr(partition))
self.debug("Everyone else: " + repr(other_nodes))
for node in partition:
if not self.CM.isolate_node(node, other_nodes):
self.logger.log("Could not isolate %s" % node)
return 0
return 1
def heal_partition(self, partition):
other_nodes = []
other_nodes.extend(self.Env["nodes"])
for node in partition:
try:
other_nodes.remove(node)
except ValueError:
self.logger.log("Node "+node+" not in " + repr(self.Env["nodes"]))
if len(other_nodes) == 0:
return 1
self.debug("Healing partition: " + repr(partition))
self.debug("Everyone else: " + repr(other_nodes))
for node in partition:
self.CM.unisolate_node(node, other_nodes)
def __call__(self, node):
'''Perform split-brain test'''
self.incr("calls")
self.passed = 1
partitions = {}
ret = self.startall(None)
if not ret:
return self.failure("Setup failed")
while 1:
# Retry until we get multiple partitions
partitions = {}
p_max = len(self.Env["nodes"])
for node in self.Env["nodes"]:
p = self.Env.RandomGen.randint(1, p_max)
if not p in partitions:
partitions[p] = []
partitions[p].append(node)
p_max = len(partitions.keys())
if p_max > 1:
break
# else, try again
self.debug("Created %d partitions" % p_max)
for key in list(partitions.keys()):
self.debug("Partition["+str(key)+"]:\t"+repr(partitions[key]))
# Disabling STONITH to reduce test complexity for now
self.rsh(node, "crm_attribute -V -n stonith-enabled -v false")
for key in list(partitions.keys()):
self.isolate_partition(partitions[key])
count = 30
while count > 0:
if len(self.CM.find_partitions()) != p_max:
time.sleep(10)
else:
break
else:
self.failure("Expected partitions were not created")
# Target number of partitions formed - wait for stability
if not self.CM.cluster_stable():
self.failure("Partitioned cluster not stable")
# Now audit the cluster state
self.CM.partitions_expected = p_max
if not self.audit():
self.failure("Audits failed")
self.CM.partitions_expected = 1
# And heal them again
for key in list(partitions.keys()):
self.heal_partition(partitions[key])
# Wait for a single partition to form
count = 30
while count > 0:
if len(self.CM.find_partitions()) != 1:
time.sleep(10)
count -= 1
else:
break
else:
self.failure("Cluster did not reform")
# Wait for it to have the right number of members
count = 30
while count > 0:
members = []
partitions = self.CM.find_partitions()
if len(partitions) > 0:
members = partitions[0].split()
if len(members) != len(self.Env["nodes"]):
time.sleep(10)
count -= 1
else:
break
else:
self.failure("Cluster did not completely reform")
# Wait up to 20 minutes - the delay is more preferable than
# trying to continue with in a messed up state
if not self.CM.cluster_stable(1200):
self.failure("Reformed cluster not stable")
if self.Env["continue"] == 1:
answer = "Y"
else:
try:
answer = raw_input('Continue? [nY]')
except EOFError, e:
answer = "n"
if answer and answer == "n":
raise ValueError("Reformed cluster not stable")
# Turn fencing back on
if self.Env["DoFencing"]:
self.rsh(node, "crm_attribute -V -D -n stonith-enabled")
self.CM.cluster_stable()
if self.passed:
return self.success()
return self.failure("See previous errors")
def errorstoignore(self):
'''Return list of errors which are 'normal' and should be ignored'''
return [
r"Another DC detected:",
r"(ERROR|error).*: .*Application of an update diff failed",
r"crmd.*:.*not in our membership list",
r"CRIT:.*node.*returning after partition",
]
def is_applicable(self):
if not self.is_applicable_common():
return 0
return len(self.Env["nodes"]) > 2
AllTestClasses.append(SplitBrainTest)
class Reattach(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "Reattach"
self.startall = SimulStartLite(cm)
self.restart1 = RestartTest(cm)
self.stopall = SimulStopLite(cm)
self.is_unsafe = 0 # Handled by canrunnow()
def _is_managed(self, node):
is_managed = self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -Q -G -d true", 1)
is_managed = is_managed[:-1] # Strip off the newline
return is_managed == "true"
def _set_unmanaged(self, node):
self.debug("Disable resource management")
self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -v false")
def _set_managed(self, node):
self.debug("Re-enable resource management")
self.rsh(node, "crm_attribute -t rsc_defaults -n is-managed -D")
def setup(self, node):
attempt = 0
if not self.startall(None):
return None
# Make sure we are really _really_ stable and that all
# resources, including those that depend on transient node
# attributes, are started
while not self.CM.cluster_stable(double_check=True):
if attempt < 5:
attempt += 1
self.debug("Not stable yet, re-testing")
else:
self.logger.log("Cluster is not stable")
return None
return 1
def teardown(self, node):
# Make sure 'node' is up
start = StartTest(self.CM)
start(node)
if not self._is_managed(node):
self.logger.log("Attempting to re-enable resource management on %s" % node)
self._set_managed(node)
self.CM.cluster_stable()
if not self._is_managed(node):
self.logger.log("Could not re-enable resource management")
return 0
return 1
def canrunnow(self, node):
'''Return TRUE if we can meaningfully run right now'''
if self.find_ocfs2_resources(node):
self.logger.log("Detach/Reattach scenarios are not possible with OCFS2 services present")
return 0
return 1
def __call__(self, node):
self.incr("calls")
pats = []
# Conveniently, pengine will display this message when disabling management,
# even if fencing is not enabled, so we can rely on it.
managed = self.create_watch(["Delaying fencing operations"], 60)
managed.setwatch()
self._set_unmanaged(node)
if not managed.lookforall():
self.logger.log("Patterns not found: " + repr(managed.unmatched))
return self.failure("Resource management not disabled")
pats = []
pats.append(self.templates["Pat:RscOpOK"] % ("start", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("stop", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("promote", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("demote", ".*"))
pats.append(self.templates["Pat:RscOpOK"] % ("migrate", ".*"))
watch = self.create_watch(pats, 60, "ShutdownActivity")
watch.setwatch()
self.debug("Shutting down the cluster")
ret = self.stopall(None)
if not ret:
self._set_managed(node)
return self.failure("Couldn't shut down the cluster")
self.debug("Bringing the cluster back up")
ret = self.startall(None)
time.sleep(5) # allow ping to update the CIB
if not ret:
self._set_managed(node)
return self.failure("Couldn't restart the cluster")
if self.local_badnews("ResourceActivity:", watch):
self._set_managed(node)
return self.failure("Resources stopped or started during cluster restart")
watch = self.create_watch(pats, 60, "StartupActivity")
watch.setwatch()
# Re-enable resource management (and verify it happened).
self._set_managed(node)
self.CM.cluster_stable()
if not self._is_managed(node):
return self.failure("Could not re-enable resource management")
# Ignore actions for STONITH resources
ignore = []
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rclass == "stonith":
self.debug("Ignoring start actions for %s" % r.id)
ignore.append(self.templates["Pat:RscOpOK"] % ("start", r.id))
if self.local_badnews("ResourceActivity:", watch, ignore):
return self.failure("Resources stopped or started after resource management was re-enabled")
return ret
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"resources were active at shutdown",
]
def is_applicable(self):
if self.Env["Name"] == "crm-lha":
return None
return 1
AllTestClasses.append(Reattach)
class SpecialTest1(CTSTest):
'''Set up a custom test to cause quorum failure issues for Andrew'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SpecialTest1"
self.startall = SimulStartLite(cm)
self.restart1 = RestartTest(cm)
self.stopall = SimulStopLite(cm)
def __call__(self, node):
'''Perform the 'SpecialTest1' test for Andrew. '''
self.incr("calls")
# Shut down all the nodes...
ret = self.stopall(None)
if not ret:
return self.failure("Could not stop all nodes")
# Test config recovery when the other nodes come up
self.rsh(node, "rm -f "+CTSvars.CRM_CONFIG_DIR+"/cib*")
# Start the selected node
ret = self.restart1(node)
if not ret:
return self.failure("Could not start "+node)
# Start all remaining nodes
ret = self.startall(None)
if not ret:
return self.failure("Could not start the remaining nodes")
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
# Errors that occur as a result of the CIB being wiped
return [
r"error.*: v1 patchset error, patch failed to apply: Application of an update diff failed",
r"error.*: Resource start-up disabled since no STONITH resources have been defined",
r"error.*: Either configure some or disable STONITH with the stonith-enabled option",
r"error.*: NOTE: Clusters with shared data need STONITH to ensure data integrity",
]
AllTestClasses.append(SpecialTest1)
class HAETest(CTSTest):
'''Set up a custom test to cause quorum failure issues for Andrew'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "HAETest"
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
self.is_loop = 1
def setup(self, node):
# Start all remaining nodes
ret = self.startall(None)
if not ret:
return self.failure("Couldn't start all nodes")
return self.success()
def teardown(self, node):
# Stop everything
ret = self.stopall(None)
if not ret:
return self.failure("Couldn't stop all nodes")
return self.success()
def wait_on_state(self, node, resource, expected_clones, attempts=240):
while attempts > 0:
active = 0
(rc, lines) = self.rsh(node, "crm_resource -r %s -W -Q" % resource, stdout=None)
# Hack until crm_resource does the right thing
if rc == 0 and lines:
active = len(lines)
if len(lines) == expected_clones:
return 1
elif rc == 1:
self.debug("Resource %s is still inactive" % resource)
elif rc == 234:
self.logger.log("Unknown resource %s" % resource)
return 0
elif rc == 246:
self.logger.log("Cluster is inactive")
return 0
elif rc != 0:
self.logger.log("Call to crm_resource failed, rc=%d" % rc)
return 0
else:
self.debug("Resource %s is active on %d times instead of %d" % (resource, active, expected_clones))
attempts -= 1
time.sleep(1)
return 0
def find_dlm(self, node):
self.r_dlm = None
(rc, lines) = self.rsh(node, "crm_resource -c", None)
for line in lines:
if re.search("^Resource", line):
r = AuditResource(self.CM, line)
if r.rtype == "controld" and r.parent != "NA":
self.debug("Found dlm: %s" % self.r_dlm)
self.r_dlm = r.parent
return 1
return 0
def find_hae_resources(self, node):
self.r_dlm = None
self.r_o2cb = None
self.r_ocfs2 = []
if self.find_dlm(node):
self.find_ocfs2_resources(node)
def is_applicable(self):
if not self.is_applicable_common():
return 0
if self.Env["Schema"] == "hae":
return 1
return None
class HAERoleTest(HAETest):
def __init__(self, cm):
'''Lars' mount/unmount test for the HA extension. '''
HAETest.__init__(self,cm)
self.name = "HAERoleTest"
def change_state(self, node, resource, target):
rc = self.rsh(node, "crm_resource -V -r %s -p target-role -v %s --meta" % (resource, target))
return rc
def __call__(self, node):
self.incr("calls")
lpc = 0
failed = 0
delay = 2
done = time.time() + self.Env["loop-minutes"]*60
self.find_hae_resources(node)
clone_max = len(self.Env["nodes"])
while time.time() <= done and not failed:
lpc = lpc + 1
self.change_state(node, self.r_dlm, "Stopped")
if not self.wait_on_state(node, self.r_dlm, 0):
self.failure("%s did not go down correctly" % self.r_dlm)
failed = lpc
self.change_state(node, self.r_dlm, "Started")
if not self.wait_on_state(node, self.r_dlm, clone_max):
self.failure("%s did not come up correctly" % self.r_dlm)
failed = lpc
if not self.wait_on_state(node, self.r_o2cb, clone_max):
self.failure("%s did not come up correctly" % self.r_o2cb)
failed = lpc
for fs in self.r_ocfs2:
if not self.wait_on_state(node, fs, clone_max):
self.failure("%s did not come up correctly" % fs)
failed = lpc
if failed:
return self.failure("iteration %d failed" % failed)
return self.success()
AllTestClasses.append(HAERoleTest)
class HAEStandbyTest(HAETest):
'''Set up a custom test to cause quorum failure issues for Andrew'''
def __init__(self, cm):
HAETest.__init__(self,cm)
self.name = "HAEStandbyTest"
def change_state(self, node, resource, target):
rc = self.rsh(node, "crm_standby -V -l reboot -v %s" % (target))
return rc
def __call__(self, node):
self.incr("calls")
lpc = 0
failed = 0
done = time.time() + self.Env["loop-minutes"]*60
self.find_hae_resources(node)
clone_max = len(self.Env["nodes"])
while time.time() <= done and not failed:
lpc = lpc + 1
self.change_state(node, self.r_dlm, "true")
if not self.wait_on_state(node, self.r_dlm, clone_max-1):
self.failure("%s did not go down correctly" % self.r_dlm)
failed = lpc
self.change_state(node, self.r_dlm, "false")
if not self.wait_on_state(node, self.r_dlm, clone_max):
self.failure("%s did not come up correctly" % self.r_dlm)
failed = lpc
if not self.wait_on_state(node, self.r_o2cb, clone_max):
self.failure("%s did not come up correctly" % self.r_o2cb)
failed = lpc
for fs in self.r_ocfs2:
if not self.wait_on_state(node, fs, clone_max):
self.failure("%s did not come up correctly" % fs)
failed = lpc
if failed:
return self.failure("iteration %d failed" % failed)
return self.success()
AllTestClasses.append(HAEStandbyTest)
class NearQuorumPointTest(CTSTest):
'''
This test brings larger clusters near the quorum point (50%).
In addition, it will test doing starts and stops at the same time.
Here is how I think it should work:
- loop over the nodes and decide randomly which will be up and which
will be down Use a 50% probability for each of up/down.
- figure out what to do to get into that state from the current state
- in parallel, bring up those going up and bring those going down.
'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "NearQuorumPoint"
def __call__(self, dummy):
'''Perform the 'NearQuorumPoint' test. '''
self.incr("calls")
startset = []
stopset = []
stonith = self.CM.prepare_fencing_watcher("NearQuorumPoint")
#decide what to do with each node
for node in self.Env["nodes"]:
action = self.Env.RandomGen.choice(["start","stop"])
#action = self.Env.RandomGen.choice(["start","stop","no change"])
if action == "start" :
startset.append(node)
elif action == "stop" :
stopset.append(node)
self.debug("start nodes:" + repr(startset))
self.debug("stop nodes:" + repr(stopset))
#add search patterns
watchpats = [ ]
for node in stopset:
if self.CM.ShouldBeStatus[node] == "up":
watchpats.append(self.templates["Pat:We_stopped"] % node)
for node in startset:
if self.CM.ShouldBeStatus[node] == "down":
#watchpats.append(self.templates["Pat:Slave_started"] % node)
watchpats.append(self.templates["Pat:Local_started"] % node)
else:
for stopping in stopset:
if self.CM.ShouldBeStatus[stopping] == "up":
watchpats.append(self.templates["Pat:They_stopped"] % (node, self.CM.key_for_node(stopping)))
if len(watchpats) == 0:
return self.skipped()
if len(startset) != 0:
watchpats.append(self.templates["Pat:DC_IDLE"])
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
#begin actions
for node in stopset:
if self.CM.ShouldBeStatus[node] == "up":
self.CM.StopaCMnoBlock(node)
for node in startset:
if self.CM.ShouldBeStatus[node] == "down":
self.CM.StartaCMnoBlock(node)
#get the result
if watch.lookforall():
self.CM.cluster_stable()
self.CM.fencing_cleanup("NearQuorumPoint", stonith)
return self.success()
self.logger.log("Warn: Patterns not found: " + repr(watch.unmatched))
#get the "bad" nodes
upnodes = []
for node in stopset:
if self.CM.StataCM(node) == 1:
upnodes.append(node)
downnodes = []
for node in startset:
if self.CM.StataCM(node) == 0:
downnodes.append(node)
self.CM.fencing_cleanup("NearQuorumPoint", stonith)
if upnodes == [] and downnodes == []:
self.CM.cluster_stable()
# Make sure they're completely down with no residule
for node in stopset:
self.rsh(node, self.templates["StopCmd"])
return self.success()
if len(upnodes) > 0:
self.logger.log("Warn: Unstoppable nodes: " + repr(upnodes))
if len(downnodes) > 0:
self.logger.log("Warn: Unstartable nodes: " + repr(downnodes))
return self.failure()
def is_applicable(self):
if self.Env["Name"] == "crm-cman":
return None
return 1
AllTestClasses.append(NearQuorumPointTest)
class RollingUpgradeTest(CTSTest):
'''Perform a rolling upgrade of the cluster'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "RollingUpgrade"
self.start = StartTest(cm)
self.stop = StopTest(cm)
self.stopall = SimulStopLite(cm)
self.startall = SimulStartLite(cm)
def setup(self, node):
# Start all remaining nodes
ret = self.stopall(None)
if not ret:
return self.failure("Couldn't stop all nodes")
for node in self.Env["nodes"]:
if not self.downgrade(node, None):
return self.failure("Couldn't downgrade %s" % node)
ret = self.startall(None)
if not ret:
return self.failure("Couldn't start all nodes")
return self.success()
def teardown(self, node):
# Stop everything
ret = self.stopall(None)
if not ret:
return self.failure("Couldn't stop all nodes")
for node in self.Env["nodes"]:
if not self.upgrade(node, None):
return self.failure("Couldn't upgrade %s" % node)
return self.success()
def install(self, node, version, start=1, flags="--force"):
target_dir = "/tmp/rpm-%s" % version
src_dir = "%s/%s" % (self.Env["rpm-dir"], version)
self.logger.log("Installing %s on %s with %s" % (version, node, flags))
if not self.stop(node):
return self.failure("stop failure: "+node)
rc = self.rsh(node, "mkdir -p %s" % target_dir)
rc = self.rsh(node, "rm -f %s/*.rpm" % target_dir)
(rc, lines) = self.rsh(node, "ls -1 %s/*.rpm" % src_dir, None)
for line in lines:
line = line[:-1]
rc = self.rsh.cp("%s" % (line), "%s:%s/" % (node, target_dir))
rc = self.rsh(node, "rpm -Uvh %s %s/*.rpm" % (flags, target_dir))
if start and not self.start(node):
return self.failure("start failure: "+node)
return self.success()
def upgrade(self, node, start=1):
return self.install(node, self.Env["current-version"], start)
def downgrade(self, node, start=1):
return self.install(node, self.Env["previous-version"], start, "--force --nodeps")
def __call__(self, node):
'''Perform the 'Rolling Upgrade' test. '''
self.incr("calls")
for node in self.Env["nodes"]:
if self.upgrade(node):
return self.failure("Couldn't upgrade %s" % node)
self.CM.cluster_stable()
return self.success()
def is_applicable(self):
if not self.is_applicable_common():
return None
if not "rpm-dir" in self.Env.keys():
return None
if not "current-version" in self.Env.keys():
return None
if not "previous-version" in self.Env.keys():
return None
return 1
# Register RestartTest as a good test to run
AllTestClasses.append(RollingUpgradeTest)
class BSC_AddResource(CTSTest):
'''Add a resource to the cluster'''
def __init__(self, cm):
CTSTest.__init__(self, cm)
self.name = "AddResource"
self.resource_offset = 0
self.cib_cmd = """cibadmin -C -o %s -X '%s' """
def __call__(self, node):
self.incr("calls")
self.resource_offset = self.resource_offset + 1
r_id = "bsc-rsc-%s-%d" % (node, self.resource_offset)
start_pat = "crmd.*%s_start_0.*confirmed.*ok"
patterns = []
patterns.append(start_pat % r_id)
watch = self.create_watch(patterns, self.Env["DeadTime"])
watch.setwatch()
ip = self.NextIP()
if not self.make_ip_resource(node, r_id, "ocf", "IPaddr", ip):
return self.failure("Make resource %s failed" % r_id)
failed = 0
watch_result = watch.lookforall()
if watch.unmatched:
for regex in watch.unmatched:
self.logger.log ("Warn: Pattern not found: %s" % (regex))
failed = 1
if failed:
return self.failure("Resource pattern(s) not found")
if not self.CM.cluster_stable(self.Env["DeadTime"]):
return self.failure("Unstable cluster")
return self.success()
def NextIP(self):
ip = self.Env["IPBase"]
if ":" in ip:
fields = ip.rpartition(":")
fields[2] = str(hex(int(fields[2], 16)+1))
print(str(hex(int(f[2], 16)+1)))
else:
fields = ip.rpartition('.')
fields[2] = str(int(fields[2])+1)
ip = fields[0] + fields[1] + fields[3];
self.Env["IPBase"] = ip
return ip.strip()
def make_ip_resource(self, node, id, rclass, type, ip):
self.logger.log("Creating %s::%s:%s (%s) on %s" % (rclass,type,id,ip,node))
rsc_xml="""
<primitive id="%s" class="%s" type="%s" provider="heartbeat">
<instance_attributes id="%s"><attributes>
<nvpair id="%s" name="ip" value="%s"/>
</attributes></instance_attributes>
</primitive>""" % (id, rclass, type, id, id, ip)
node_constraint = """
<rsc_location id="run_%s" rsc="%s">
<rule id="pref_run_%s" score="100">
<expression id="%s_loc_expr" attribute="#uname" operation="eq" value="%s"/>
</rule>
</rsc_location>""" % (id, id, id, id, node)
rc = 0
(rc, lines) = self.rsh(node, self.cib_cmd % ("constraints", node_constraint), None)
if rc != 0:
self.logger.log("Constraint creation failed: %d" % rc)
return None
(rc, lines) = self.rsh(node, self.cib_cmd % ("resources", rsc_xml), None)
if rc != 0:
self.logger.log("Resource creation failed: %d" % rc)
return None
return 1
def is_applicable(self):
if self.Env["DoBSC"]:
return 1
return None
AllTestClasses.append(BSC_AddResource)
class SimulStopLite(CTSTest):
'''Stop any active nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStopLite"
def __call__(self, dummy):
'''Perform the 'SimulStopLite' setup work. '''
self.incr("calls")
self.debug("Setup: " + self.name)
# We ignore the "node" parameter...
watchpats = [ ]
for node in self.Env["nodes"]:
if self.CM.ShouldBeStatus[node] == "up":
self.incr("WasStarted")
watchpats.append(self.templates["Pat:We_stopped"] % node)
#if self.Env["use_logd"]:
# watchpats.append(self.templates["Pat:Logd_stopped"] % node)
if len(watchpats) == 0:
self.CM.clear_all_caches()
return self.success()
# Stop all the nodes - at about the same time...
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
self.set_timer()
for node in self.Env["nodes"]:
if self.CM.ShouldBeStatus[node] == "up":
self.CM.StopaCMnoBlock(node)
if watch.lookforall():
self.CM.clear_all_caches()
# Make sure they're completely down with no residule
for node in self.Env["nodes"]:
self.rsh(node, self.templates["StopCmd"])
return self.success()
did_fail = 0
up_nodes = []
for node in self.Env["nodes"]:
if self.CM.StataCM(node) == 1:
did_fail = 1
up_nodes.append(node)
if did_fail:
return self.failure("Active nodes exist: " + repr(up_nodes))
self.logger.log("Warn: All nodes stopped but CTS didnt detect: "
+ repr(watch.unmatched))
self.CM.clear_all_caches()
return self.failure("Missing log message: "+repr(watch.unmatched))
def is_applicable(self):
'''SimulStopLite is a setup test and never applicable'''
return 0
class SimulStartLite(CTSTest):
'''Start any stopped nodes ~ simultaneously'''
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "SimulStartLite"
def __call__(self, dummy):
'''Perform the 'SimulStartList' setup work. '''
self.incr("calls")
self.debug("Setup: " + self.name)
# We ignore the "node" parameter...
node_list = []
for node in self.Env["nodes"]:
if self.CM.ShouldBeStatus[node] == "down":
self.incr("WasStopped")
node_list.append(node)
self.set_timer()
while len(node_list) > 0:
# Repeat until all nodes come up
watchpats = [ ]
uppat = self.templates["Pat:Slave_started"]
if self.CM.upcount() == 0:
uppat = self.templates["Pat:Local_started"]
watchpats.append(self.templates["Pat:DC_IDLE"])
for node in node_list:
watchpats.append(uppat % node)
watchpats.append(self.templates["Pat:InfraUp"] % node)
watchpats.append(self.templates["Pat:PacemakerUp"] % node)
# Start all the nodes - at about the same time...
watch = self.create_watch(watchpats, self.Env["DeadTime"]+10)
watch.setwatch()
stonith = self.CM.prepare_fencing_watcher(self.name)
for node in node_list:
self.CM.StartaCMnoBlock(node)
watch.lookforall()
node_list = self.CM.fencing_cleanup(self.name, stonith)
if node_list == None:
return self.failure("Cluster did not stabilize")
# Remove node_list messages from watch.unmatched
for node in node_list:
self.logger.debug("Dealing with stonith operations for %s" % repr(node_list))
if watch.unmatched:
try:
watch.unmatched.remove(uppat % node)
except:
self.debug("Already matched: %s" % (uppat % node))
try:
watch.unmatched.remove(self.templates["Pat:InfraUp"] % node)
except:
self.debug("Already matched: %s" % (self.templates["Pat:InfraUp"] % node))
try:
watch.unmatched.remove(self.templates["Pat:PacemakerUp"] % node)
except:
self.debug("Already matched: %s" % (self.templates["Pat:PacemakerUp"] % node))
if watch.unmatched:
for regex in watch.unmatched:
self.logger.log ("Warn: Startup pattern not found: %s" %(regex))
if not self.CM.cluster_stable():
return self.failure("Cluster did not stabilize")
did_fail = 0
unstable = []
for node in self.Env["nodes"]:
if self.CM.StataCM(node) == 0:
did_fail = 1
unstable.append(node)
if did_fail:
return self.failure("Unstarted nodes exist: " + repr(unstable))
unstable = []
for node in self.Env["nodes"]:
if not self.CM.node_stable(node):
did_fail = 1
unstable.append(node)
if did_fail:
return self.failure("Unstable cluster nodes exist: " + repr(unstable))
return self.success()
def is_applicable(self):
'''SimulStartLite is a setup test and never applicable'''
return 0
def TestList(cm, audits):
result = []
for testclass in AllTestClasses:
bound_test = testclass(cm)
if bound_test.is_applicable():
bound_test.Audits = audits
result.append(bound_test)
return result
class RemoteLXC(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = "RemoteLXC"
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.num_containers = 2
self.is_container = 1
self.is_docker_unsafe = 1
self.failed = 0
self.fail_string = ""
def start_lxc_simple(self, node):
# restore any artifacts laying around from a previous test.
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
# generate the containers, put them in the config, add some resources to them
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc1"))
pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc2"))
pats.append(self.templates["Pat:RscOpOK"] % ("start", "lxc-ms"))
pats.append(self.templates["Pat:RscOpOK"] % ("promote", "lxc-ms"))
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)
self.set_timer("remoteSimpleInit")
watch.lookforall()
self.log_timer("remoteSimpleInit")
if watch.unmatched:
self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
self.failed = 1
def cleanup_lxc_simple(self, node):
pats = [ ]
# if the test failed, attempt to clean up the cib and libvirt environment
# as best as possible
if self.failed == 1:
# restore libvirt and cib
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
return
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container1"))
pats.append(self.templates["Pat:RscOpOK"] % ("stop", "container2"))
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")
self.set_timer("remoteSimpleCleanup")
watch.lookforall()
self.log_timer("remoteSimpleCleanup")
if watch.unmatched:
self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))
self.failed = 1
# cleanup libvirt
self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -s -R &>/dev/null")
def __call__(self, node):
'''Perform the 'RemoteLXC' test. '''
self.incr("calls")
ret = self.startall(None)
if not ret:
return self.failure("Setup failed, start all nodes failed.")
rc = self.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")
if rc == 1:
self.log("Environment test for lxc support failed.")
return self.skipped()
self.start_lxc_simple(node)
self.cleanup_lxc_simple(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed == 1:
return self.failure(self.fail_string)
return self.success()
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [
r"Updating failcount for ping",
r"pengine.*: Recover (ping|lxc-ms|container)\s*\(.*\)",
# The orphaned lxc-ms resource causes an expected transition error
# that is a result of the pengine not having knowledge that the
# ms resource used to be a clone. As a result it looks like that
# resource is running in multiple locations when it shouldn't... But in
# this instance we know why this error is occurring and that it is expected.
r"Calculated [Tt]ransition .* /var/lib/pacemaker/pengine/pe-error",
r"Resource lxc-ms .* is active on 2 nodes attempting recovery",
r"Unknown operation: fail",
r"(ERROR|error): sending stonithRA op to stonithd failed.",
r"VirtualDomain.*ERROR: Unable to determine emulator",
]
AllTestClasses.append(RemoteLXC)
class RemoteDriver(CTSTest):
def __init__(self, cm):
CTSTest.__init__(self,cm)
self.name = self.__class__.__name__
self.is_docker_unsafe = 1
self.start = StartTest(cm)
self.startall = SimulStartLite(cm)
self.stop = StopTest(cm)
self.remote_rsc = "remote-rsc"
self.cib_cmd = """cibadmin -C -o %s -X '%s' """
self.reset()
def reset(self):
self.pcmk_started = 0
self.failed = False
self.fail_string = ""
self.remote_node_added = 0
self.remote_rsc_added = 0
self.remote_use_reconnect_interval = self.Env.RandomGen.choice([True,False])
def fail(self, msg):
""" Mark test as failed. """
self.failed = True
# Always log the failure.
self.logger.log(msg)
# Use first failure as test status, as it's likely to be most useful.
if not self.fail_string:
self.fail_string = msg
def get_othernode(self, node):
for othernode in self.Env["nodes"]:
if othernode == node:
# we don't want to try and use the cib that we just shutdown.
# find a cluster node that is not our soon to be remote-node.
continue
else:
return othernode
def del_rsc(self, node, rsc):
othernode = self.get_othernode(node)
rc = self.rsh(othernode, "crm_resource -D -r %s -t primitive" % (rsc))
if rc != 0:
self.fail("Removal of resource '%s' failed" % rsc)
def add_rsc(self, node, rsc_xml):
othernode = self.get_othernode(node)
rc = self.rsh(othernode, self.cib_cmd % ("resources", rsc_xml))
if rc != 0:
self.fail("resource creation failed")
def add_primitive_rsc(self, node):
rsc_xml = """
<primitive class="ocf" id="%s" provider="heartbeat" type="Dummy">
<operations>
<op id="remote-rsc-monitor-interval-10s" interval="10s" name="monitor"/>
</operations>
<meta_attributes id="remote-meta_attributes"/>
</primitive>""" % (self.remote_rsc)
self.add_rsc(node, rsc_xml)
if not self.failed:
self.remote_rsc_added = 1
def add_connection_rsc(self, node):
if self.remote_use_reconnect_interval:
# use reconnect interval and make sure to set cluster-recheck-interval as well.
rsc_xml = """
<primitive class="ocf" id="%s" provider="pacemaker" type="remote">
<instance_attributes id="remote-instance_attributes"/>
<instance_attributes id="remote-instance_attributes">
<nvpair id="remote-instance_attributes-server" name="server" value="%s"/>
<nvpair id="remote-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
</instance_attributes>
<operations>
<op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
<op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="60"/>
</operations>
</primitive>""" % (self.remote_node, node)
self.rsh(self.get_othernode(node), self.templates["SetCheckInterval"] % ("45s"))
else:
# not using reconnect interval
rsc_xml = """
<primitive class="ocf" id="%s" provider="pacemaker" type="remote">
<instance_attributes id="remote-instance_attributes"/>
<instance_attributes id="remote-instance_attributes">
<nvpair id="remote-instance_attributes-server" name="server" value="%s"/>
</instance_attributes>
<operations>
<op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
<op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="120"/>
</operations>
</primitive>""" % (self.remote_node, node)
self.add_rsc(node, rsc_xml)
if not self.failed:
self.remote_node_added = 1
def stop_pcmk_remote(self, node):
# disable pcmk remote
for i in range(10):
rc = self.rsh(node, "service pacemaker_remote stop")
if rc != 0:
time.sleep(6)
else:
break
def start_pcmk_remote(self, node):
for i in range(10):
rc = self.rsh(node, "service pacemaker_remote start")
if rc != 0:
time.sleep(6)
else:
self.pcmk_started = 1
break
def kill_pcmk_remote(self, node):
""" Simulate a Pacemaker Remote daemon failure. """
# We kill the process to prevent a graceful stop,
# then stop it to prevent the OS from restarting it.
self.rsh(node, "killall -9 pacemaker_remoted")
self.stop_pcmk_remote(node)
def start_metal(self, node):
pcmk_started = 0
# make sure the resource doesn't already exist for some reason
self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_rsc))
self.rsh(node, "crm_resource -D -r %s -t primitive" % (self.remote_node))
if not self.stop(node):
self.fail("Failed to shutdown cluster node %s" % node)
return
self.start_pcmk_remote(node)
if self.pcmk_started == 0:
self.fail("Failed to start pacemaker_remote on node %s" % node)
return
# convert node to baremetal node now that it has shutdow the cluster stack
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
pats.append(self.templates["Pat:DC_IDLE"])
self.add_connection_rsc(node)
self.set_timer("remoteMetalInit")
watch.lookforall()
self.log_timer("remoteMetalInit")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
def migrate_connection(self, node):
if self.failed:
return
pats = [ ]
pats.append(self.templates["Pat:RscOpOK"] % ("migrate_to", self.remote_node))
pats.append(self.templates["Pat:RscOpOK"] % ("migrate_from", self.remote_node))
pats.append(self.templates["Pat:DC_IDLE"])
watch = self.create_watch(pats, 120)
watch.setwatch()
(rc, lines) = self.rsh(node, "crm_resource -M -r %s" % (self.remote_node), None)
if rc != 0:
self.fail("failed to move remote node connection resource")
return
self.set_timer("remoteMetalMigrate")
watch.lookforall()
self.log_timer("remoteMetalMigrate")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
return
def fail_rsc(self, node):
if self.failed:
return
watchpats = [ ]
watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("stop", self.remote_rsc, self.remote_node))
watchpats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
watchpats.append(self.templates["Pat:DC_IDLE"])
watch = self.create_watch(watchpats, 120)
watch.setwatch()
self.debug("causing dummy rsc to fail.")
rc = self.rsh(node, "rm -f /var/run/resource-agents/Dummy*")
self.set_timer("remoteRscFail")
watch.lookforall()
self.log_timer("remoteRscFail")
if watch.unmatched:
self.fail("Unmatched patterns during rsc fail: %s" % watch.unmatched)
def fail_connection(self, node):
if self.failed:
return
watchpats = [ ]
watchpats.append(self.templates["Pat:FenceOpOK"] % self.remote_node)
watchpats.append(self.templates["Pat:NodeFenced"] % self.remote_node)
watch = self.create_watch(watchpats, 120)
watch.setwatch()
# force stop the pcmk remote daemon. this will result in fencing
self.debug("Force stopped active remote node")
self.kill_pcmk_remote(node)
self.debug("Waiting for remote node to be fenced.")
self.set_timer("remoteMetalFence")
watch.lookforall()
self.log_timer("remoteMetalFence")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
return
self.debug("Waiting for the remote node to come back up")
self.CM.ns.WaitForNodeToComeUp(node, 120);
pats = [ ]
watch = self.create_watch(pats, 240)
watch.setwatch()
pats.append(self.templates["Pat:RscOpOK"] % ("start", self.remote_node))
if self.remote_rsc_added == 1:
pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
# start the remote node again watch it integrate back into cluster.
self.start_pcmk_remote(node)
if self.pcmk_started == 0:
self.fail("Failed to start pacemaker_remote on node %s" % node)
return
self.debug("Waiting for remote node to rejoin cluster after being fenced.")
self.set_timer("remoteMetalRestart")
watch.lookforall()
self.log_timer("remoteMetalRestart")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
return
def add_dummy_rsc(self, node):
if self.failed:
return
# verify we can put a resource on the remote node
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
pats.append(self.templates["Pat:RscRemoteOpOK"] % ("start", self.remote_rsc, self.remote_node))
pats.append(self.templates["Pat:DC_IDLE"])
# Add a resource that must live on remote-node
self.add_primitive_rsc(node)
# force that rsc to prefer the remote node.
(rc, line) = self.CM.rsh(node, "crm_resource -M -r %s -N %s -f" % (self.remote_rsc, self.remote_node), None)
if rc != 0:
self.fail("Failed to place remote resource on remote node.")
return
self.set_timer("remoteMetalRsc")
watch.lookforall()
self.log_timer("remoteMetalRsc")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
def test_attributes(self, node):
if self.failed:
return
# This verifies permanent attributes can be set on a remote-node. It also
- # verifies the remote-node can edit it's own cib node section remotely.
+ # verifies the remote-node can edit its own cib node section remotely.
(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -v testval -N %s" % (self.remote_node), None)
if rc != 0:
self.fail("Failed to set remote-node attribute. rc:%s output:%s" % (rc, line))
return
(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -Q -N %s" % (self.remote_node), None)
if rc != 0:
self.fail("Failed to get remote-node attribute")
return
(rc, line) = self.CM.rsh(node, "crm_attribute -l forever -n testattr -D -N %s" % (self.remote_node), None)
if rc != 0:
self.fail("Failed to delete remote-node attribute")
return
def cleanup_metal(self, node):
if self.pcmk_started == 0:
return
pats = [ ]
watch = self.create_watch(pats, 120)
watch.setwatch()
if self.remote_rsc_added == 1:
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_rsc))
if self.remote_node_added == 1:
pats.append(self.templates["Pat:RscOpOK"] % ("stop", self.remote_node))
self.set_timer("remoteMetalCleanup")
if self.remote_use_reconnect_interval:
self.debug("Cleaning up re-check interval")
self.rsh(self.get_othernode(node), self.templates["ClearCheckInterval"])
if self.remote_rsc_added == 1:
# Remove dummy resource added for remote node tests
self.debug("Cleaning up dummy rsc put on remote node")
self.rsh(node, "crm_resource -U -r %s" % self.remote_rsc)
self.del_rsc(node, self.remote_rsc)
if self.remote_node_added == 1:
# Remove remote node's connection resource
self.debug("Cleaning up remote node connection resource")
self.rsh(node, "crm_resource -U -r %s" % (self.remote_node))
self.del_rsc(node, self.remote_node)
watch.lookforall()
self.log_timer("remoteMetalCleanup")
if watch.unmatched:
self.fail("Unmatched patterns: %s" % watch.unmatched)
self.stop_pcmk_remote(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.remote_node_added == 1:
# Remove remote node itself
self.debug("Cleaning up node entry for remote node")
self.rsh(self.get_othernode(node), "crm_node --force --remove %s" % self.remote_node)
def setup_env(self, node):
self.remote_node = "remote_%s" % (node)
# we are assuming if all nodes have a key, that it is
# the right key... If any node doesn't have a remote
# key, we regenerate it everywhere.
if self.rsh.exists_on_all("/etc/pacemaker/authkey", self.Env["nodes"]):
return
# create key locally
(handle, keyfile) = tempfile.mkstemp(".cts")
os.close(handle)
devnull = open(os.devnull, 'wb')
subprocess.check_call(["dd", "if=/dev/urandom", "of=%s" % keyfile, "bs=4096", "count=1"],
stdout=devnull, stderr=devnull)
devnull.close()
# sync key throughout the cluster
for node in self.Env["nodes"]:
self.rsh(node, "mkdir -p --mode=0750 /etc/pacemaker")
self.rsh.cp(keyfile, "root@%s:/etc/pacemaker/authkey" % node)
self.rsh(node, "chgrp haclient /etc/pacemaker /etc/pacemaker/authkey")
self.rsh(node, "chmod 0640 /etc/pacemaker/authkey")
os.unlink(keyfile)
def is_applicable(self):
if not self.is_applicable_common():
return False
for node in self.Env["nodes"]:
rc = self.rsh(node, "type pacemaker_remoted >/dev/null 2>&1")
if rc != 0:
return False
return True
def start_new_test(self, node):
self.incr("calls")
self.reset()
ret = self.startall(None)
if not ret:
return self.failure("Setup failed, start all nodes failed.")
self.setup_env(node)
self.start_metal(node)
self.add_dummy_rsc(node)
def __call__(self, node):
return self.failure("This base class is not meant to be called directly.")
def errorstoignore(self):
'''Return list of errors which should be ignored'''
return [ """is running on remote.*which isn't allowed""",
"""Connection terminated""",
"""Failed to send remote""",
]
# RemoteDriver is just a base class for other tests, so it is not added to AllTestClasses
class RemoteBasic(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteBaremetal' test. '''
self.start_new_test(node)
self.test_attributes(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
AllTestClasses.append(RemoteBasic)
class RemoteStonithd(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteStonithd' test. '''
self.start_new_test(node)
self.fail_connection(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
def is_applicable(self):
if not RemoteDriver.is_applicable(self):
return False
if "DoFencing" in self.Env.keys():
return self.Env["DoFencing"]
return True
def errorstoignore(self):
ignore_pats = [
r"Unexpected disconnect on remote-node",
r"crmd.*:\s+error.*: Operation remote_.*_monitor",
r"crmd.*:\s+error.*: Result of monitor operation for remote_.*",
r"pengine.*:\s+Recover remote_.*\s*\(.*\)",
r"Calculated [Tt]ransition .* /var/lib/pacemaker/pengine/pe-error",
r"error.*: Resource .*ocf::.* is active on 2 nodes attempting recovery",
]
ignore_pats.extend(RemoteDriver.errorstoignore(self))
return ignore_pats
AllTestClasses.append(RemoteStonithd)
class RemoteMigrate(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteMigrate' test. '''
self.start_new_test(node)
self.migrate_connection(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
AllTestClasses.append(RemoteMigrate)
class RemoteRscFailure(RemoteDriver):
def __call__(self, node):
'''Perform the 'RemoteRscFailure' test. '''
self.start_new_test(node)
# This is an important step. We are migrating the connection
# before failing the resource. This verifies that the migration
# has properly maintained control over the remote-node.
self.migrate_connection(node)
self.fail_rsc(node)
self.cleanup_metal(node)
self.debug("Waiting for the cluster to recover")
self.CM.cluster_stable()
if self.failed:
return self.failure(self.fail_string)
return self.success()
def errorstoignore(self):
ignore_pats = [
r"pengine.*: Recover remote-rsc\s*\(.*\)",
]
ignore_pats.extend(RemoteDriver.errorstoignore(self))
return ignore_pats
AllTestClasses.append(RemoteRscFailure)
# vim:ts=4:sw=4:et:
diff --git a/fencing/fence_dummy b/fencing/fence_dummy
index 6c0864c915..c3c4604131 100755
--- a/fencing/fence_dummy
+++ b/fencing/fence_dummy
@@ -1,362 +1,362 @@
#!/usr/bin/python
# The Following Agent Has Been Tested On:
#
# Virsh 0.3.3 on RHEL 5.2 with xen-3.0.3-51
#
import sys, time, random, os, atexit, getopt, re
#BEGIN_VERSION_GENERATION
RELEASE_VERSION="3.1.6"
BUILD_DATE="(built Mon Oct 24 12:14:08 UTC 2011)"
REDHAT_COPYRIGHT="Copyright (C) Red Hat, Inc. 2004-2010 All rights reserved."
#END_VERSION_GENERATION
all_opt = {
"help" : {
"getopt" : "h",
"longopt" : "help",
"help" : "-h, --help Display this help and exit",
"required" : "0",
"shortdesc" : "Display help and exit",
"order" : 54 },
"version" : {
"getopt" : "V",
"longopt" : "version",
"help" : "-V, --version Output version information and exit",
"required" : "0",
"shortdesc" : "Display version information and exit",
"order" : 53 },
"quiet" : {
"getopt" : "q",
"help" : "",
"order" : 50 },
"verbose" : {
"getopt" : "v",
"longopt" : "verbose",
"help" : "-v, --verbose Verbose mode",
"required" : "0",
"shortdesc" : "Verbose mode",
"order" : 51 },
"debug" : {
"getopt" : "D:",
"longopt" : "debug-file",
"help" : "-D, --debug-file=[debugfile] Debugging to output file",
"required" : "0",
"shortdesc" : "Write debug information to given file",
"order" : 52 },
"random_sleep_range": {
"getopt" : "R:",
"required" : "0",
"longopt" : "random_sleep_range",
"help" : "--random_sleep-range=[seconds] Issue a sleep between 1 and [seconds]. Used for testing.",
"shortdesc" : "Issue a sleep between 1 and [seconds]",
"order" : 1 },
"mode": {
"getopt" : "M:",
"longopt" : "mode",
"required" : "0",
"help" : "--mode=(pass|fail|random). Used for testing.",
"shortdesc" : "Should operations always pass, always fail or fail at random",
"order" : 1 },
"delay" : {
"getopt" : "f:",
"longopt" : "delay",
"help" : "--delay [seconds] Wait X seconds before fencing is started",
"required" : "0",
"shortdesc" : "Wait X seconds before fencing is started",
"default" : "0",
"order" : 200 },
"action" : {
"getopt" : "o:",
"longopt" : "action",
"help" : "-o, --action=[action] Action: status, list, reboot (default), off or on",
"required" : "1",
"shortdesc" : "Fencing Action",
"default" : "reboot",
"order" : 1 },
"port" : {
"getopt" : "n:",
"longopt" : "plug",
"help" : "-n, --plug=[id] Physical plug number on device or\n" +
" name of virtual machine",
"required" : "1",
"shortdesc" : "Physical plug number or name of virtual machine",
"order" : 1 },
"switch" : {
"getopt" : "s:",
"longopt" : "switch",
"help" : "-s, --switch=[id] Physical switch number on device",
"required" : "0",
"shortdesc" : "Physical switch number on device",
"order" : 1 },
"nodeid" : {
"getopt" : "i:",
"longopt" : "nodeid",
"help" : "-i, --nodeid corosync id of fence victim",
"required" : "0",
"shortdesc" : "The corosync id of fence victim",
"order" : 1},
"uuid" : {
"getopt" : "U:",
"longopt" : "uuid",
"help" : "-U, --uuid UUID of the VM to fence.",
"required" : "0",
"shortdesc" : "The UUID of the virtual machine to fence.",
"order" : 1},
"mock_dynamic_hosts" : {
"getopt" : "H:",
"longopt" : "mock_dynamic_hosts",
"help" : "-H, --mock_dynamic_hosts=[hostlist] List of hosts we can fence.",
"required" : "0",
"shortdesc" : "A list of hosts we can fence.",
"order" : 1}
}
def show_docs(options, docs = None):
device_opt = options["device_opt"]
if docs == None:
docs = { }
docs["shortdesc"] = "Fence agent"
docs["longdesc"] = ""
## Process special options (and exit)
#####
if options.has_key("-h"):
usage(device_opt)
sys.exit(0)
if options.has_key("-o") and options["-o"].lower() == "metadata":
sys.stderr.write("asked for fence_dummy metadata\n")
metadata(device_opt, options, docs)
sys.exit(0)
if options.has_key("-V"):
print __main__.RELEASE_VERSION, __main__.BUILD_DATE
print __main__.REDHAT_COPYRIGHT
sys.exit(0)
def usage(avail_opt):
global all_opt
print "Usage:"
print "\t" + os.path.basename(sys.argv[0]) + " [options]"
print "Options:"
sorted_list = [ (key, all_opt[key]) for key in avail_opt ]
sorted_list.sort(lambda x, y: cmp(x[1]["order"], y[1]["order"]))
for key, value in sorted_list:
if len(value["help"]) != 0:
print " " + value["help"]
def metadata(avail_opt, options, docs):
global all_opt
sorted_list = [ (key, all_opt[key]) for key in avail_opt ]
sorted_list.sort(lambda x, y: cmp(x[1]["order"], y[1]["order"]))
print "<?xml version=\"1.0\" ?>"
print "<resource-agent name=\"" + os.path.basename(sys.argv[0]) + "\" shortdesc=\"" + docs["shortdesc"] + "\" >"
print "<longdesc>" + docs["longdesc"] + "</longdesc>"
if docs.has_key("vendorurl"):
print "<vendor-url>" + docs["vendorurl"] + "</vendor-url>"
print "<parameters>"
for option, value in sorted_list:
if all_opt[option].has_key("shortdesc"):
print "\t<parameter name=\"" + option + "\" unique=\"0\" required=\"" + all_opt[option]["required"] + "\">"
default = ""
if all_opt[option].has_key("default"):
default = "default=\""+str(all_opt[option]["default"])+"\""
elif options.has_key("-" + all_opt[option]["getopt"][:-1]):
if options["-" + all_opt[option]["getopt"][:-1]]:
try:
default = "default=\"" + options["-" + all_opt[option]["getopt"][:-1]] + "\""
except TypeError:
## @todo/@note: Currently there is no clean way how to handle lists
## we can create a string from it but we can't set it on command line
default = "default=\"" + str(options["-" + all_opt[option]["getopt"][:-1]]) +"\""
elif options.has_key("-" + all_opt[option]["getopt"]):
default = "default=\"true\" "
mixed = all_opt[option]["help"]
## split it between option and help text
res = re.compile("^(.*--\S+)\s+", re.IGNORECASE | re.S).search(mixed)
if (None != res):
mixed = res.group(1)
mixed = mixed.replace("<", "<").replace(">", ">")
print "\t\t<getopt mixed=\"" + mixed + "\" />"
if all_opt[option]["getopt"].count(":") > 0:
print "\t\t<content type=\"string\" "+default+" />"
else:
print "\t\t<content type=\"boolean\" "+default+" />"
print "\t\t<shortdesc lang=\"en\">" + all_opt[option]["shortdesc"] + "</shortdesc>"
print "\t</parameter>"
print "</parameters>"
print "<actions>"
if avail_opt.count("io_fencing") == 0:
print "\t<action name=\"on\" on_target=\"1\" />"
print "\t<action name=\"off\" />"
print "\t<action name=\"reboot\" />"
else:
print "\t<action name=\"enable\" />"
print "\t<action name=\"disable\" />"
print "\t<action name=\"status\" />"
print "\t<action name=\"monitor\" />"
print "\t<action name=\"metadata\" />"
print "\t<action name=\"list\" />"
print "</actions>"
print "</resource-agent>"
def process_input(avail_opt):
global all_opt
##
## Set standard environment
#####
os.putenv("LANG", "C")
os.putenv("LC_ALL", "C")
##
## Prepare list of options for getopt
#####
getopt_string = ""
longopt_list = [ ]
for k in avail_opt:
if all_opt.has_key(k):
getopt_string += all_opt[k]["getopt"]
else:
fail_usage("Parse error: unknown option '"+k+"'")
if all_opt.has_key(k) and all_opt[k].has_key("longopt"):
if all_opt[k]["getopt"].endswith(":"):
longopt_list.append(all_opt[k]["longopt"] + "=")
else:
longopt_list.append(all_opt[k]["longopt"])
## Compatibility layer
if avail_opt.count("module_name") == 1:
getopt_string += "n:"
longopt_list.append("plug=")
##
## Read options from command line or standard input
#####
if len(sys.argv) > 1:
try:
opt, args = getopt.gnu_getopt(sys.argv[1:], getopt_string, longopt_list)
except getopt.GetoptError, error:
fail_usage("Parse error: " + error.msg)
## Transform longopt to short one which are used in fencing agents
#####
old_opt = opt
opt = { }
for o in dict(old_opt).keys():
if o.startswith("--"):
for x in all_opt.keys():
if all_opt[x].has_key("longopt") and "--" + all_opt[x]["longopt"] == o:
opt["-" + all_opt[x]["getopt"].rstrip(":")] = dict(old_opt)[o]
else:
opt[o] = dict(old_opt)[o]
## Compatibility Layer
#####
z = dict(opt)
if z.has_key("-T") == 1:
z["-o"] = "status"
if z.has_key("-n") == 1:
z["-m"] = z["-n"]
opt = z
##
#####
else:
opt = { }
name = ""
for line in sys.stdin.readlines():
line = line.strip()
if ((line.startswith("#")) or (len(line) == 0)):
continue
(name, value) = (line + "=").split("=", 1)
value = value[:-1]
## Compatibility Layer
######
if name == "option":
name = "action"
##
######
if avail_opt.count(name) == 0:
sys.stderr.write("Parse error: Ignoring unknown option '"+line+"'\n")
continue
if all_opt[name]["getopt"].endswith(":"):
opt["-"+all_opt[name]["getopt"].rstrip(":")] = value
elif ((value == "1") or (value.lower() == "yes") or (value.lower() == "on") or (value.lower() == "true")):
opt["-"+all_opt[name]["getopt"]] = "1"
return opt
def atexit_handler():
try:
sys.stdout.close()
os.close(1)
except IOError:
sys.stderr.write("%s failed to close standard output\n"%(sys.argv[0]))
sys.exit(1)
def main():
global all_opt
device_opt = all_opt.keys()
## Defaults for fence agent
docs = { }
docs["shortdesc"] = "Dummy fence agent"
- docs["longdesc"] = "fence_dummy is a fake fencing agent which reports success based on it's mode (pass|fail|random) without doing anything."
+ docs["longdesc"] = "fence_dummy is a fake fencing agent which reports success based on its mode (pass|fail|random) without doing anything."
atexit.register(atexit_handler)
options = process_input(device_opt)
options["device_opt"] = device_opt
show_docs(options, docs)
# random sleep for testing
if options.has_key("-f"):
val = int(options["-f"])
sys.stderr.write("delay sleep for %d seconds\n" % val)
time.sleep(val)
if options.has_key("-R"):
val = int(options["-R"])
ran = random.randint(1, val)
sys.stderr.write("random sleep for %d seconds\n" % ran)
time.sleep(ran)
if options.has_key("-o") and (options["-o"] == "monitor"):
sys.stderr.write("fence_dummy monitor called\n")
sys.exit(0)
if options.has_key("-o") and (options["-o"] == "list"):
sys.stderr.write("fence_dummy action (list) called\n")
if options.has_key("-H"):
print options["-H"]
else:
sys.stderr.write("were asked for hostlist but attribute mock_dynamic_hosts wasn't set\n")
if options.has_key("-M"):
if options["-M"] == "pass":
sys.exit(0)
elif options["-M"] == "fail":
sys.exit(1)
sys.exit(random.randint(0, 1))
if __name__ == "__main__":
main()
diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
index d546d7d16f..5e3b191d70 100644
--- a/include/crm/pengine/status.h
+++ b/include/crm/pengine/status.h
@@ -1,434 +1,434 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef PENGINE_STATUS__H
# define PENGINE_STATUS__H
# include <glib.h>
# include <crm/common/iso8601.h>
# include <crm/pengine/common.h>
typedef struct node_s pe_node_t;
typedef struct node_s node_t;
typedef struct pe_action_s action_t;
typedef struct pe_action_s pe_action_t;
typedef struct resource_s resource_t;
typedef struct ticket_s ticket_t;
typedef enum no_quorum_policy_e {
no_quorum_freeze,
no_quorum_stop,
no_quorum_ignore,
no_quorum_suicide
} no_quorum_policy_t;
enum node_type {
node_ping,
node_member,
node_remote
};
enum pe_restart {
pe_restart_restart,
pe_restart_ignore
};
enum pe_find {
pe_find_renamed = 0x001,
pe_find_clone = 0x004,
pe_find_current = 0x008,
pe_find_inactive = 0x010,
};
# define pe_flag_have_quorum 0x00000001ULL
# define pe_flag_symmetric_cluster 0x00000002ULL
# define pe_flag_is_managed_default 0x00000004ULL
# define pe_flag_maintenance_mode 0x00000008ULL
# define pe_flag_stonith_enabled 0x00000010ULL
# define pe_flag_have_stonith_resource 0x00000020ULL
# define pe_flag_enable_unfencing 0x00000040ULL
# define pe_flag_concurrent_fencing 0x00000080ULL
# define pe_flag_stop_rsc_orphans 0x00000100ULL
# define pe_flag_stop_action_orphans 0x00000200ULL
# define pe_flag_stop_everything 0x00000400ULL
# define pe_flag_start_failure_fatal 0x00001000ULL
# define pe_flag_remove_after_stop 0x00002000ULL
# define pe_flag_startup_probes 0x00010000ULL
# define pe_flag_have_status 0x00020000ULL
# define pe_flag_have_remote_nodes 0x00040000ULL
# define pe_flag_quick_location 0x00100000ULL
# define pe_flag_sanitized 0x00200000ULL
typedef struct pe_working_set_s {
xmlNode *input;
crm_time_t *now;
/* options extracted from the input */
char *dc_uuid;
node_t *dc_node;
const char *stonith_action;
const char *placement_strategy;
unsigned long long flags;
int stonith_timeout;
int default_resource_stickiness;
no_quorum_policy_t no_quorum_policy;
GHashTable *config_hash;
GHashTable *tickets;
GHashTable *singletons; /* Actions for which there can be only one - ie. fence nodeX */
GListPtr nodes;
GListPtr resources;
GListPtr placement_constraints;
GListPtr ordering_constraints;
GListPtr colocation_constraints;
GListPtr ticket_constraints;
GListPtr actions;
xmlNode *failed;
xmlNode *op_defaults;
xmlNode *rsc_defaults;
/* stats */
int num_synapse;
int max_valid_nodes;
int order_id;
int action_id;
/* final output */
xmlNode *graph;
GHashTable *template_rsc_sets;
const char *localhost;
GHashTable *tags;
int blocked_resources;
int disabled_resources;
} pe_working_set_t;
struct node_shared_s {
const char *id;
const char *uname;
/* Make all these flags into a bitfield one day */
gboolean online;
gboolean standby;
gboolean standby_onfail;
gboolean pending;
gboolean unclean;
gboolean unseen;
gboolean shutdown;
gboolean expected_up;
gboolean is_dc;
int num_resources;
GListPtr running_rsc; /* resource_t* */
GListPtr allocated_rsc; /* resource_t* */
resource_t *remote_rsc;
GHashTable *attrs; /* char* => char* */
enum node_type type;
GHashTable *utilization;
/*! cache of calculated rsc digests for this node. */
GHashTable *digest_cache;
gboolean maintenance;
gboolean rsc_discovery_enabled;
gboolean remote_requires_reset;
gboolean remote_was_fenced;
};
struct node_s {
int weight;
gboolean fixed;
int count;
struct node_shared_s *details;
int rsc_discover_mode;
};
# include <crm/pengine/complex.h>
# define pe_rsc_orphan 0x00000001ULL
# define pe_rsc_managed 0x00000002ULL
# define pe_rsc_block 0x00000004ULL /* Further operations are prohibited due to failure policy */
# define pe_rsc_orphan_container_filler 0x00000008ULL
# define pe_rsc_notify 0x00000010ULL
# define pe_rsc_unique 0x00000020ULL
# define pe_rsc_fence_device 0x00000040ULL
# define pe_rsc_provisional 0x00000100ULL
# define pe_rsc_allocating 0x00000200ULL
# define pe_rsc_merging 0x00000400ULL
# define pe_rsc_munging 0x00000800ULL
# define pe_rsc_try_reload 0x00001000ULL
# define pe_rsc_reload 0x00002000ULL
# define pe_rsc_failed 0x00010000ULL
# define pe_rsc_shutdown 0x00020000ULL
# define pe_rsc_runnable 0x00040000ULL
# define pe_rsc_start_pending 0x00080000ULL
# define pe_rsc_starting 0x00100000ULL
# define pe_rsc_stopping 0x00200000ULL
# define pe_rsc_migrating 0x00400000ULL
# define pe_rsc_allow_migrate 0x00800000ULL
# define pe_rsc_failure_ignored 0x01000000ULL
# define pe_rsc_unexpectedly_running 0x02000000ULL
# define pe_rsc_maintenance 0x04000000ULL
# define pe_rsc_needs_quorum 0x10000000ULL
# define pe_rsc_needs_fencing 0x20000000ULL
# define pe_rsc_needs_unfencing 0x40000000ULL
# define pe_rsc_have_unfencing 0x80000000ULL /* obsolete (not set or used by cluster) */
enum pe_graph_flags {
pe_graph_none = 0x00000,
pe_graph_updated_first = 0x00001,
pe_graph_updated_then = 0x00002,
pe_graph_disable = 0x00004,
};
/* *INDENT-OFF* */
enum pe_action_flags {
pe_action_pseudo = 0x00001,
pe_action_runnable = 0x00002,
pe_action_optional = 0x00004,
pe_action_print_always = 0x00008,
pe_action_have_node_attrs = 0x00010,
pe_action_failure_is_fatal = 0x00020, /* no longer used, here for API compatibility */
pe_action_implied_by_stonith = 0x00040,
pe_action_migrate_runnable = 0x00080,
pe_action_dumped = 0x00100,
pe_action_processed = 0x00200,
pe_action_clear = 0x00400,
pe_action_dangle = 0x00800,
pe_action_requires_any = 0x01000, /* This action requires one or mre of its dependencies to be runnable
* We use this to clear the runnable flag before checking dependencies
*/
pe_action_reschedule = 0x02000,
pe_action_tracking = 0x04000,
};
/* *INDENT-ON* */
struct resource_s {
char *id;
char *clone_name;
xmlNode *xml;
xmlNode *orig_xml;
xmlNode *ops_xml;
resource_t *parent;
void *variant_opaque;
enum pe_obj_types variant;
resource_object_functions_t *fns;
resource_alloc_functions_t *cmds;
enum rsc_recovery_type recovery_type;
enum pe_restart restart_type;
int priority;
int stickiness;
int sort_index;
int failure_timeout;
int effective_priority;
int migration_threshold;
gboolean is_remote_node;
unsigned long long flags;
GListPtr rsc_cons_lhs; /* rsc_colocation_t* */
GListPtr rsc_cons; /* rsc_colocation_t* */
GListPtr rsc_location; /* rsc_to_node_t* */
GListPtr actions; /* action_t* */
GListPtr rsc_tickets; /* rsc_ticket* */
node_t *allocated_to;
GListPtr running_on; /* node_t* */
GHashTable *known_on; /* node_t* */
GHashTable *allowed_nodes; /* node_t* */
enum rsc_role_e role;
enum rsc_role_e next_role;
GHashTable *meta;
GHashTable *parameters;
GHashTable *utilization;
GListPtr children; /* resource_t* */
GListPtr dangling_migrations; /* node_t* */
node_t *partial_migration_target;
node_t *partial_migration_source;
resource_t *container;
GListPtr fillers;
char *pending_task;
const char *isolation_wrapper;
gboolean exclusive_discover;
int remote_reconnect_interval;
pe_working_set_t *cluster;
xmlNode *versioned_parameters;
};
struct pe_action_s {
int id;
int priority;
resource_t *rsc;
node_t *node;
xmlNode *op_entry;
char *task;
char *uuid;
char *cancel_task;
enum pe_action_flags flags;
enum rsc_start_requirement needs;
enum action_fail_response on_fail;
enum rsc_role_e fail_role;
action_t *pre_notify;
action_t *pre_notified;
action_t *post_notify;
action_t *post_notified;
int seen_count;
GHashTable *meta;
GHashTable *extra;
/*
* These two varables are associated with the constraint logic
* that involves first having one or more actions runnable before
* then allowing this action to execute.
*
* These varables are used with features such as 'clone-min' which
* requires at minimum X number of cloned instances to be running
* before an order dependency can run. Another option that uses
* this is 'require-all=false' in ordering constrants. This option
* says "only required one instance of a resource to start before
* allowing dependencies to start" basicall require-all=false is
* the same as clone-min=1.
*/
/* current number of known runnable actions in the before list. */
int runnable_before;
/* the number of "before" runnable actions required for this action
* to be considered runnable */
int required_runnable_before;
GListPtr actions_before; /* action_warpper_t* */
GListPtr actions_after; /* action_warpper_t* */
};
struct ticket_s {
char *id;
gboolean granted;
time_t last_granted;
gboolean standby;
GHashTable *state;
};
typedef struct tag_s {
char *id;
GListPtr refs;
} tag_t;
enum pe_link_state {
pe_link_not_dumped,
pe_link_dumped,
pe_link_dup,
};
/* *INDENT-OFF* */
enum pe_ordering {
pe_order_none = 0x0, /* deleted */
pe_order_optional = 0x1, /* pure ordering, nothing implied */
pe_order_apply_first_non_migratable = 0x2, /* Only apply this constraint's ordering if first is not migratable. */
pe_order_implies_first = 0x10, /* If 'then' is required, ensure 'first' is too */
pe_order_implies_then = 0x20, /* If 'first' is required, ensure 'then' is too */
pe_order_implies_first_master = 0x40, /* Imply 'first' is required when 'then' is required and then's rsc holds Master role. */
/* first requires then to be both runnable and migrate runnable. */
pe_order_implies_first_migratable = 0x80,
pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */
pe_order_pseudo_left = 0x200, /* 'then' can only be pseudo if 'first' is runnable */
pe_order_implies_then_on_node = 0x400, /* If 'first' is required on 'nodeX',
* ensure instances of 'then' on 'nodeX' are too.
* Only really useful if 'then' is a clone and 'first' is not
*/
pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */
pe_order_stonith_stop = 0x2000, /* only applies if the action is non-pseudo */
pe_order_serialize_only = 0x4000, /* serialize */
pe_order_same_node = 0x8000, /* applies only if 'first' and 'then' are on same node */
pe_order_implies_first_printed = 0x10000, /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not mandatory */
pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */
pe_order_load = 0x200000, /* Only relevant if... */
- pe_order_one_or_more = 0x400000, /* 'then' is only runnable if one or more of it's dependencies are too */
+ pe_order_one_or_more = 0x400000, /* 'then' is runnable only if one or more of its dependencies are too */
pe_order_anti_colocation = 0x800000,
pe_order_preserve = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
pe_order_trace = 0x4000000, /* test marker */
};
/* *INDENT-ON* */
typedef struct action_wrapper_s action_wrapper_t;
struct action_wrapper_s {
enum pe_ordering type;
enum pe_link_state state;
action_t *action;
};
const char *rsc_printable_id(resource_t *rsc);
gboolean cluster_status(pe_working_set_t * data_set);
void set_working_set_defaults(pe_working_set_t * data_set);
void cleanup_calculations(pe_working_set_t * data_set);
resource_t *pe_find_resource(GListPtr rsc_list, const char *id_rh);
node_t *pe_find_node(GListPtr node_list, const char *uname);
node_t *pe_find_node_id(GListPtr node_list, const char *id);
node_t *pe_find_node_any(GListPtr node_list, const char *id, const char *uname);
GListPtr find_operations(const char *rsc, const char *node, gboolean active_filter,
pe_working_set_t * data_set);
#endif
diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
index 633c50ded2..1908401230 100644
--- a/lib/cib/cib_utils.c
+++ b/lib/cib/cib_utils.c
@@ -1,834 +1,834 @@
/*
* Copyright (c) 2004 International Business Machines
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#include <crm_internal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <sys/utsname.h>
#include <glib.h>
#include <crm/crm.h>
#include <crm/cib/internal.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/pengine/rules.h>
struct config_root_s {
const char *name;
const char *parent;
const char *path;
};
/*
* "//crm_config" will also work in place of "/cib/configuration/crm_config"
* The / prefix means find starting from the root, whereas the // prefix means
* find anywhere and risks multiple matches
*/
/* *INDENT-OFF* */
struct config_root_s known_paths[] = {
{ NULL, NULL, "//cib" },
{ XML_TAG_CIB, NULL, "//cib" },
{ XML_CIB_TAG_STATUS, "/cib", "//cib/status" },
{ XML_CIB_TAG_CONFIGURATION,"/cib", "//cib/configuration" },
{ XML_CIB_TAG_CRMCONFIG, "/cib/configuration", "//cib/configuration/crm_config" },
{ XML_CIB_TAG_NODES, "/cib/configuration", "//cib/configuration/nodes" },
{ XML_CIB_TAG_DOMAINS, "/cib/configuration", "//cib/configuration/domains" },
{ XML_CIB_TAG_RESOURCES, "/cib/configuration", "//cib/configuration/resources" },
{ XML_CIB_TAG_CONSTRAINTS, "/cib/configuration", "//cib/configuration/constraints" },
{ XML_CIB_TAG_OPCONFIG, "/cib/configuration", "//cib/configuration/op_defaults" },
{ XML_CIB_TAG_RSCCONFIG, "/cib/configuration", "//cib/configuration/rsc_defaults" },
{ XML_CIB_TAG_ACLS, "/cib/configuration", "//cib/configuration/acls" },
{ XML_TAG_FENCING_TOPOLOGY, "/cib/configuration", "//cib/configuration/fencing-topology" },
{ XML_CIB_TAG_SECTION_ALL, NULL, "//cib" },
};
/* *INDENT-ON* */
int
cib_compare_generation(xmlNode * left, xmlNode * right)
{
int lpc = 0;
const char *attributes[] = {
XML_ATTR_GENERATION_ADMIN,
XML_ATTR_GENERATION,
XML_ATTR_NUMUPDATES,
};
crm_log_xml_trace(left, "left");
crm_log_xml_trace(right, "right");
for (lpc = 0; lpc < DIMOF(attributes); lpc++) {
int int_elem_l = -1;
int int_elem_r = -1;
const char *elem_r = NULL;
const char *elem_l = crm_element_value(left, attributes[lpc]);
if (right != NULL) {
elem_r = crm_element_value(right, attributes[lpc]);
}
if (elem_l != NULL) {
int_elem_l = crm_parse_int(elem_l, NULL);
}
if (elem_r != NULL) {
int_elem_r = crm_parse_int(elem_r, NULL);
}
if (int_elem_l < int_elem_r) {
crm_trace("%s (%s < %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
return -1;
} else if (int_elem_l > int_elem_r) {
crm_trace("%s (%s > %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
return 1;
}
}
return 0;
}
/* Deprecated - doesn't expose -EACCES */
xmlNode *
get_cib_copy(cib_t * cib)
{
xmlNode *xml_cib;
int options = cib_scope_local | cib_sync_call;
int rc = pcmk_ok;
if (cib->state == cib_disconnected) {
return NULL;
}
rc = cib->cmds->query(cib, NULL, &xml_cib, options);
if (rc == -EACCES) {
return NULL;
} else if (rc != pcmk_ok) {
crm_err("Couldn't retrieve the CIB");
free_xml(xml_cib);
return NULL;
} else if (xml_cib == NULL) {
crm_err("The CIB result was empty");
free_xml(xml_cib);
return NULL;
}
if (safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
return xml_cib;
}
free_xml(xml_cib);
return NULL;
}
xmlNode *
cib_get_generation(cib_t * cib)
{
xmlNode *the_cib = NULL;
xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
cib->cmds->query(cib, NULL, &the_cib, cib_scope_local | cib_sync_call);
if (the_cib != NULL) {
copy_in_properties(generation, the_cib);
free_xml(the_cib);
}
return generation;
}
gboolean
cib_version_details(xmlNode * cib, int *admin_epoch, int *epoch, int *updates)
{
*epoch = -1;
*updates = -1;
*admin_epoch = -1;
if (cib == NULL) {
return FALSE;
} else {
crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
}
return TRUE;
}
gboolean
cib_diff_version_details(xmlNode * diff, int *admin_epoch, int *epoch, int *updates,
int *_admin_epoch, int *_epoch, int *_updates)
{
int add[] = { 0, 0, 0 };
int del[] = { 0, 0, 0 };
xml_patch_versions(diff, add, del);
*admin_epoch = add[0];
*epoch = add[1];
*updates = add[2];
*_admin_epoch = del[0];
*_epoch = del[1];
*_updates = del[2];
return TRUE;
}
/*
* The caller should never free the return value
*/
const char *
get_object_path(const char *object_type)
{
int lpc = 0;
int max = DIMOF(known_paths);
for (; lpc < max; lpc++) {
if ((object_type == NULL && known_paths[lpc].name == NULL)
|| safe_str_eq(object_type, known_paths[lpc].name)) {
return known_paths[lpc].path;
}
}
return NULL;
}
const char *
get_object_parent(const char *object_type)
{
int lpc = 0;
int max = DIMOF(known_paths);
for (; lpc < max; lpc++) {
if (safe_str_eq(object_type, known_paths[lpc].name)) {
return known_paths[lpc].parent;
}
}
return NULL;
}
xmlNode *
get_object_root(const char *object_type, xmlNode * the_root)
{
const char *xpath = get_object_path(object_type);
if (xpath == NULL) {
return the_root; /* or return NULL? */
}
return get_xpath_object(xpath, the_root, LOG_DEBUG_4);
}
/*
* It is the callers responsibility to free both the new CIB (output)
* and the new CIB (input)
*/
xmlNode *
createEmptyCib(int admin_epoch)
{
xmlNode *cib_root = NULL, *config = NULL;
cib_root = create_xml_node(NULL, XML_TAG_CIB);
crm_xml_add(cib_root, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
crm_xml_add(cib_root, XML_ATTR_VALIDATION, xml_latest_schema());
crm_xml_add_int(cib_root, XML_ATTR_GENERATION, admin_epoch);
crm_xml_add_int(cib_root, XML_ATTR_NUMUPDATES, 0);
crm_xml_add_int(cib_root, XML_ATTR_GENERATION_ADMIN, 0);
config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
create_xml_node(cib_root, XML_CIB_TAG_STATUS);
create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
create_xml_node(config, XML_CIB_TAG_NODES);
create_xml_node(config, XML_CIB_TAG_RESOURCES);
create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);
return cib_root;
}
static bool
cib_acl_enabled(xmlNode *xml, const char *user)
{
bool rc = FALSE;
#if ENABLE_ACL
if(pcmk_acl_required(user)) {
const char *value = NULL;
GHashTable *options = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
cib_read_config(options, xml);
value = cib_pref(options, "enable-acl");
rc = crm_is_true(value);
g_hash_table_destroy(options);
}
crm_debug("CIB ACL is %s", rc ? "enabled" : "disabled");
#endif
return rc;
}
int
cib_perform_op(const char *op, int call_options, cib_op_t * fn, gboolean is_query,
const char *section, xmlNode * req, xmlNode * input,
gboolean manage_counters, gboolean * config_changed,
xmlNode * current_cib, xmlNode ** result_cib, xmlNode ** diff, xmlNode ** output)
{
int rc = pcmk_ok;
gboolean check_dtd = TRUE;
xmlNode *top = NULL;
xmlNode *scratch = NULL;
xmlNode *local_diff = NULL;
const char *new_version = NULL;
static struct qb_log_callsite *diff_cs = NULL;
const char *user = crm_element_value(req, F_CIB_USER);
bool with_digest = FALSE;
crm_trace("Begin %s%s%s op", is_set(call_options, cib_dryrun)?"dry-run of ":"", is_query ? "read-only " : "", op);
CRM_CHECK(output != NULL, return -ENOMSG);
CRM_CHECK(result_cib != NULL, return -ENOMSG);
CRM_CHECK(config_changed != NULL, return -ENOMSG);
if(output) {
*output = NULL;
}
*result_cib = NULL;
*config_changed = FALSE;
if (fn == NULL) {
return -EINVAL;
}
if (is_query) {
xmlNode *cib_ro = current_cib;
xmlNode *cib_filtered = NULL;
if(cib_acl_enabled(cib_ro, user)) {
if(xml_acl_filtered_copy(user, current_cib, current_cib, &cib_filtered)) {
if (cib_filtered == NULL) {
crm_debug("Pre-filtered the entire cib");
return -EACCES;
}
cib_ro = cib_filtered;
crm_log_xml_trace(cib_ro, "filtered");
}
}
rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);
if(output == NULL || *output == NULL) {
/* nothing */
} else if(cib_filtered == *output) {
cib_filtered = NULL; /* Let them have this copy */
} else if(*output == current_cib) {
/* They already know not to free it */
} else if(cib_filtered && (*output)->doc == cib_filtered->doc) {
/* We're about to free the document of which *output is a part */
*output = copy_xml(*output);
} else if((*output)->doc == current_cib->doc) {
/* Give them a copy they can free */
*output = copy_xml(*output);
}
free_xml(cib_filtered);
return rc;
}
if (is_set(call_options, cib_zero_copy)) {
/* Conditional on v2 patch style */
scratch = current_cib;
/* Create a shallow copy of current_cib for the version details */
current_cib = create_xml_node(NULL, (const char *)scratch->name);
copy_in_properties(current_cib, scratch);
top = current_cib;
xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);
} else {
scratch = copy_xml(current_cib);
xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
rc = (*fn) (op, call_options, section, req, input, current_cib, &scratch, output);
if(scratch && xml_tracking_changes(scratch) == FALSE) {
crm_trace("Inferring changes after %s op", op);
xml_track_changes(scratch, user, current_cib, cib_acl_enabled(current_cib, user));
xml_calculate_changes(current_cib, scratch);
}
CRM_CHECK(current_cib != scratch, return -EINVAL);
}
xml_acl_disable(scratch); /* Allow the system to make any additional changes */
if (rc == pcmk_ok && scratch == NULL) {
rc = -EINVAL;
goto done;
} else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
crm_trace("ACL rejected part or all of the proposed changes");
rc = -EACCES;
goto done;
} else if (rc != pcmk_ok) {
goto done;
}
if (scratch) {
new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION);
if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
crm_err("Discarding update with feature set '%s' greater than our own '%s'",
new_version, CRM_FEATURE_SET);
rc = -EPROTONOSUPPORT;
goto done;
}
}
if (current_cib) {
int old = 0;
int new = 0;
crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);
if (old > new) {
crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
XML_ATTR_GENERATION_ADMIN, old, new, call_options);
crm_log_xml_warn(req, "Bad Op");
crm_log_xml_warn(input, "Bad Data");
rc = -pcmk_err_old_data;
} else if (old == new) {
crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
if (old > new) {
crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
XML_ATTR_GENERATION, old, new, call_options);
crm_log_xml_warn(req, "Bad Op");
crm_log_xml_warn(input, "Bad Data");
rc = -pcmk_err_old_data;
}
}
}
crm_trace("Massaging CIB contents");
strip_text_nodes(scratch);
fix_plus_plus_recursive(scratch);
if (is_set(call_options, cib_zero_copy)) {
/* At this point, current_cib is just the 'cib' tag and its properties,
*
* The v1 format would barf on this, but we know the v2 patch
* format only needs it for the top-level version fields
*/
local_diff = xml_create_patchset(2, current_cib, scratch, (bool*)config_changed, manage_counters);
} else {
static time_t expires = 0;
time_t tm_now = time(NULL);
if (expires < tm_now) {
expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */
with_digest = TRUE;
}
local_diff = xml_create_patchset(0, current_cib, scratch, (bool*)config_changed, manage_counters);
}
xml_log_changes(LOG_TRACE, __FUNCTION__, scratch);
xml_accept_changes(scratch);
if (diff_cs == NULL) {
diff_cs = qb_log_callsite_get(__PRETTY_FUNCTION__, __FILE__, "diff-validation", LOG_DEBUG, __LINE__, crm_trace_nonlog);
}
if(local_diff) {
patchset_process_digest(local_diff, current_cib, scratch, with_digest);
xml_log_patchset(LOG_INFO, __FUNCTION__, local_diff);
crm_log_xml_trace(local_diff, "raw patch");
}
if (is_not_set(call_options, cib_zero_copy) /* The original to compare against doesn't exist */
&& local_diff
&& crm_is_callsite_active(diff_cs, LOG_TRACE, 0)) {
/* Validate the calculated patch set */
int test_rc, format = 1;
xmlNode * c = copy_xml(current_cib);
crm_element_value_int(local_diff, "format", &format);
test_rc = xml_apply_patchset(c, local_diff, manage_counters);
if(test_rc != pcmk_ok) {
save_xml_to_file(c, "PatchApply:calculated", NULL);
save_xml_to_file(current_cib, "PatchApply:input", NULL);
save_xml_to_file(scratch, "PatchApply:actual", NULL);
save_xml_to_file(local_diff, "PatchApply:diff", NULL);
crm_err("v%d patchset error, patch failed to apply: %s (%d)", format, pcmk_strerror(test_rc), test_rc);
}
free_xml(c);
}
if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
/* Throttle the amount of costly validation we perform due to status updates
* a) we don't really care whats in the status section
- * b) we don't validate any of it's contents at the moment anyway
+ * b) we don't validate any of its contents at the moment anyway
*/
check_dtd = FALSE;
}
/* === scratch must not be modified after this point ===
* Exceptions, anything in:
static filter_t filter[] = {
{ 0, XML_ATTR_ORIGIN },
{ 0, XML_CIB_ATTR_WRITTEN },
{ 0, XML_ATTR_UPDATE_ORIG },
{ 0, XML_ATTR_UPDATE_CLIENT },
{ 0, XML_ATTR_UPDATE_USER },
};
*/
if (*config_changed && is_not_set(call_options, cib_no_mtime)) {
char *now_str = NULL;
time_t now = time(NULL);
const char *schema = crm_element_value(scratch, XML_ATTR_VALIDATION);
now_str = ctime(&now);
now_str[24] = EOS; /* replace the newline */
crm_xml_replace(scratch, XML_CIB_ATTR_WRITTEN, now_str);
if (schema) {
static int minimum_schema = 0;
int current_schema = get_schema_version(schema);
if (minimum_schema == 0) {
minimum_schema = get_schema_version("pacemaker-1.2");
}
/* Does the CIB support the "update-*" attributes... */
if (current_schema >= minimum_schema) {
const char *origin = crm_element_value(req, F_ORIG);
CRM_LOG_ASSERT(origin != NULL);
crm_xml_replace(scratch, XML_ATTR_UPDATE_ORIG, origin);
crm_xml_replace(scratch, XML_ATTR_UPDATE_CLIENT,
crm_element_value(req, F_CIB_CLIENTNAME));
#if ENABLE_ACL
crm_xml_replace(scratch, XML_ATTR_UPDATE_USER, crm_element_value(req, F_CIB_USER));
#endif
}
}
}
crm_trace("Perform validation: %s", check_dtd ? "true" : "false");
if (rc == pcmk_ok && check_dtd && validate_xml(scratch, NULL, TRUE) == FALSE) {
const char *current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);
crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
rc = -pcmk_err_schema_validation;
}
done:
*result_cib = scratch;
#if ENABLE_ACL
if(rc != pcmk_ok && cib_acl_enabled(current_cib, user)) {
if(xml_acl_filtered_copy(user, current_cib, scratch, result_cib)) {
if (*result_cib == NULL) {
crm_debug("Pre-filtered the entire cib result");
}
free_xml(scratch);
}
}
#endif
if(diff) {
*diff = local_diff;
} else {
free_xml(local_diff);
}
free_xml(top);
crm_trace("Done");
return rc;
}
xmlNode *
cib_create_op(int call_id, const char *token, const char *op, const char *host, const char *section,
xmlNode * data, int call_options, const char *user_name)
{
xmlNode *op_msg = create_xml_node(NULL, "cib_command");
CRM_CHECK(op_msg != NULL, return NULL);
CRM_CHECK(token != NULL, return NULL);
crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");
crm_xml_add(op_msg, F_TYPE, T_CIB);
crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
crm_xml_add(op_msg, F_CIB_OPERATION, op);
crm_xml_add(op_msg, F_CIB_HOST, host);
crm_xml_add(op_msg, F_CIB_SECTION, section);
crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
#if ENABLE_ACL
if (user_name) {
crm_xml_add(op_msg, F_CIB_USER, user_name);
}
#endif
crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);
if (data != NULL) {
add_message_xml(op_msg, F_CIB_CALLDATA, data);
}
if (call_options & cib_inhibit_bcast) {
CRM_CHECK((call_options & cib_scope_local), return NULL);
}
return op_msg;
}
void
cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
{
xmlNode *output = NULL;
cib_callback_client_t *blob = NULL;
if (msg != NULL) {
crm_element_value_int(msg, F_CIB_RC, &rc);
crm_element_value_int(msg, F_CIB_CALLID, &call_id);
output = get_message_xml(msg, F_CIB_CALLDATA);
}
blob = g_hash_table_lookup(cib_op_callback_table, GINT_TO_POINTER(call_id));
if (blob == NULL) {
crm_trace("No callback found for call %d", call_id);
}
if (cib == NULL) {
crm_debug("No cib object supplied");
}
if (rc == -pcmk_err_diff_resync) {
/* This is an internal value that clients do not and should not care about */
rc = pcmk_ok;
}
if (blob && blob->callback && (rc == pcmk_ok || blob->only_success == FALSE)) {
crm_trace("Invoking callback %s for call %d", crm_str(blob->id), call_id);
blob->callback(msg, call_id, rc, output, blob->user_data);
} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
crm_warn("CIB command failed: %s", pcmk_strerror(rc));
crm_log_xml_debug(msg, "Failed CIB Update");
}
/* This may free user_data, so do it after the callback */
if (blob) {
remove_cib_op_callback(call_id, FALSE);
}
if (cib && cib->op_callback != NULL) {
crm_trace("Invoking global callback for call %d", call_id);
cib->op_callback(msg, call_id, rc, output);
}
crm_trace("OP callback activated for %d", call_id);
}
void
cib_native_notify(gpointer data, gpointer user_data)
{
xmlNode *msg = user_data;
cib_notify_client_t *entry = data;
const char *event = NULL;
if (msg == NULL) {
crm_warn("Skipping callback - NULL message");
return;
}
event = crm_element_value(msg, F_SUBTYPE);
if (entry == NULL) {
crm_warn("Skipping callback - NULL callback client");
return;
} else if (entry->callback == NULL) {
crm_warn("Skipping callback - NULL callback");
return;
} else if (safe_str_neq(entry->event, event)) {
crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
return;
}
crm_trace("Invoking callback for %p/%s event...", entry, event);
entry->callback(event, msg);
crm_trace("Callback invoked...");
}
pe_cluster_option cib_opts[] = {
/* name, old-name, validate, default, description */
{"enable-acl", NULL, "boolean", NULL, "false", &check_boolean,
"Enable CIB ACL", NULL}
,
};
void
cib_metadata(void)
{
config_metadata("Cluster Information Base", "1.0",
"Cluster Information Base Options",
"This is a fake resource that details the options that can be configured for the Cluster Information Base.",
cib_opts, DIMOF(cib_opts));
}
void
verify_cib_options(GHashTable * options)
{
verify_all_options(options, cib_opts, DIMOF(cib_opts));
}
const char *
cib_pref(GHashTable * options, const char *name)
{
return get_cluster_pref(options, cib_opts, DIMOF(cib_opts), name);
}
gboolean
cib_read_config(GHashTable * options, xmlNode * current_cib)
{
xmlNode *config = NULL;
crm_time_t *now = NULL;
if (options == NULL || current_cib == NULL) {
return FALSE;
}
now = crm_time_new(NULL);
g_hash_table_remove_all(options);
config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib);
if (config) {
unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options,
CIB_OPTIONS_FIRST, FALSE, now);
}
verify_cib_options(options);
crm_time_free(now);
return TRUE;
}
int
cib_apply_patch_event(xmlNode * event, xmlNode * input, xmlNode ** output, int level)
{
int rc = pcmk_err_generic;
xmlNode *diff = NULL;
CRM_ASSERT(event);
CRM_ASSERT(input);
CRM_ASSERT(output);
crm_element_value_int(event, F_CIB_RC, &rc);
diff = get_message_xml(event, F_CIB_UPDATE_RESULT);
if (rc < pcmk_ok || diff == NULL) {
return rc;
}
if (level > LOG_CRIT) {
xml_log_patchset(level, "Config update", diff);
}
if (input != NULL) {
rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output, NULL);
if (rc != pcmk_ok) {
crm_debug("Update didn't apply: %s (%d) %p", pcmk_strerror(rc), rc, *output);
if (rc == -pcmk_err_old_data) {
crm_trace("Masking error, we already have the supplied update");
return pcmk_ok;
}
free_xml(*output); *output = NULL;
return rc;
}
}
return rc;
}
gboolean
cib_internal_config_changed(xmlNode * diff)
{
gboolean changed = FALSE;
xmlXPathObject *xpathObj = NULL;
if (diff == NULL) {
return FALSE;
}
xpathObj = xpath_search(diff, "//" XML_CIB_TAG_CRMCONFIG);
if (numXpathResults(xpathObj) > 0) {
changed = TRUE;
}
freeXpathObject(xpathObj);
return changed;
}
int
cib_internal_op(cib_t * cib, const char *op, const char *host,
const char *section, xmlNode * data,
xmlNode ** output_data, int call_options, const char *user_name)
{
int (*delegate) (cib_t * cib, const char *op, const char *host,
const char *section, xmlNode * data,
xmlNode ** output_data, int call_options, const char *user_name) =
cib->delegate_fn;
#if ENABLE_ACL
if(user_name == NULL) {
user_name = getenv("CIB_user");
}
#endif
return delegate(cib, op, host, section, data, output_data, call_options, user_name);
}
diff --git a/lib/cluster/election.c b/lib/cluster/election.c
index 59f2f0c388..032091a9ee 100644
--- a/lib/cluster/election.c
+++ b/lib/cluster/election.c
@@ -1,527 +1,527 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/mainloop.h>
#include <crm/cluster/internal.h>
#include <crm/cluster/election.h>
#include <crm/crm.h>
#define STORM_INTERVAL 2 /* in seconds */
#define STORM_MULTIPLIER 5 /* multiplied by the number of nodes */
struct election_s
{
enum election_result state;
guint count;
char *name;
char *uname;
GSourceFunc cb;
GHashTable *voted;
mainloop_timer_t *timeout; /* When to stop if not everyone casts a vote */
};
static void election_complete(election_t *e)
{
crm_info("Election %s complete", e->name);
e->state = election_won;
if(e->cb) {
e->cb(e);
}
election_reset(e);
}
static gboolean election_timer_cb(gpointer user_data)
{
election_t *e = user_data;
crm_info("Election %s %p timed out", e->name, e);
election_complete(e);
return FALSE;
}
enum election_result
election_state(election_t *e)
{
if(e) {
return e->state;
}
return election_error;
}
election_t *
election_init(const char *name, const char *uname, guint period_ms, GSourceFunc cb)
{
static guint count = 0;
election_t *e = calloc(1, sizeof(election_t));
if(e != NULL) {
if(name) {
e->name = crm_strdup_printf("election-%s", name);
} else {
e->name = crm_strdup_printf("election-%u", count++);
}
e->cb = cb;
e->uname = strdup(uname);
e->timeout = mainloop_timer_add(e->name, period_ms, FALSE, election_timer_cb, e);
crm_trace("Created %s %p", e->name, e);
}
return e;
}
void
election_remove(election_t *e, const char *uname)
{
if(e && uname && e->voted) {
g_hash_table_remove(e->voted, uname);
}
}
void
election_reset(election_t *e)
{
crm_trace("Resetting election %s", e->name);
if(e) {
mainloop_timer_stop(e->timeout);
}
if (e && e->voted) {
crm_trace("Destroying voted cache with %d members", g_hash_table_size(e->voted));
g_hash_table_destroy(e->voted);
e->voted = NULL;
}
}
void
election_fini(election_t *e)
{
if(e) {
election_reset(e);
crm_trace("Destroying %s", e->name);
mainloop_timer_del(e->timeout);
free(e->uname);
free(e->name);
free(e);
}
}
static void
election_timeout_start(election_t *e)
{
if(e) {
mainloop_timer_start(e->timeout);
}
}
void
election_timeout_stop(election_t *e)
{
if(e) {
mainloop_timer_stop(e->timeout);
}
}
void
election_timeout_set_period(election_t *e, guint period)
{
if(e) {
mainloop_timer_set_period(e->timeout, period);
} else {
crm_err("No election defined");
}
}
static int
crm_uptime(struct timeval *output)
{
static time_t expires = 0;
static struct rusage info;
time_t tm_now = time(NULL);
if (expires < tm_now) {
int rc = 0;
info.ru_utime.tv_sec = 0;
info.ru_utime.tv_usec = 0;
rc = getrusage(RUSAGE_SELF, &info);
output->tv_sec = 0;
output->tv_usec = 0;
if (rc < 0) {
crm_perror(LOG_ERR, "Could not calculate the current uptime");
expires = 0;
return -1;
}
crm_debug("Current CPU usage is: %lds, %ldus", (long)info.ru_utime.tv_sec,
(long)info.ru_utime.tv_usec);
}
expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */
output->tv_sec = info.ru_utime.tv_sec;
output->tv_usec = info.ru_utime.tv_usec;
return 1;
}
static int
crm_compare_age(struct timeval your_age)
{
struct timeval our_age;
crm_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */
if (our_age.tv_sec > your_age.tv_sec) {
crm_debug("Win: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
return 1;
} else if (our_age.tv_sec < your_age.tv_sec) {
- crm_debug("Loose: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
+ crm_debug("Lose: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
return -1;
} else if (our_age.tv_usec > your_age.tv_usec) {
crm_debug("Win: %ld.%ld vs %ld.%ld (usec)",
(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
return 1;
} else if (our_age.tv_usec < your_age.tv_usec) {
- crm_debug("Loose: %ld.%ld vs %ld.%ld (usec)",
+ crm_debug("Lose: %ld.%ld vs %ld.%ld (usec)",
(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
return -1;
}
return 0;
}
void
election_vote(election_t *e)
{
struct timeval age;
xmlNode *vote = NULL;
crm_node_t *our_node;
if(e == NULL) {
crm_trace("Not voting in election: not initialized");
return;
}
our_node = crm_get_peer(0, e->uname);
if (our_node == NULL || crm_is_peer_active(our_node) == FALSE) {
crm_trace("Cannot vote yet: %p", our_node);
return;
}
e->state = election_in_progress;
vote = create_request(CRM_OP_VOTE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
e->count++;
crm_xml_add(vote, F_CRM_ELECTION_OWNER, our_node->uuid);
crm_xml_add_int(vote, F_CRM_ELECTION_ID, e->count);
crm_uptime(&age);
crm_xml_add_int(vote, F_CRM_ELECTION_AGE_S, age.tv_sec);
crm_xml_add_int(vote, F_CRM_ELECTION_AGE_US, age.tv_usec);
send_cluster_message(NULL, crm_msg_crmd, vote, TRUE);
free_xml(vote);
crm_debug("Started election %d", e->count);
if (e->voted) {
g_hash_table_destroy(e->voted);
e->voted = NULL;
}
election_timeout_start(e);
return;
}
bool
election_check(election_t *e)
{
int voted_size = 0;
int num_members = crm_active_peers();
if(e == NULL) {
crm_trace("not initialized");
return FALSE;
}
if (e->voted) {
voted_size = g_hash_table_size(e->voted);
}
/* in the case of #voted > #members, it is better to
* wait for the timeout and give the cluster time to
* stabilize
*/
if (voted_size >= num_members) {
/* we won and everyone has voted */
election_timeout_stop(e);
if (voted_size > num_members) {
GHashTableIter gIter;
const crm_node_t *node;
char *key = NULL;
g_hash_table_iter_init(&gIter, crm_peer_cache);
while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) {
if (crm_is_peer_active(node)) {
crm_err("member: %s proc=%.32x", node->uname, node->processes);
}
}
g_hash_table_iter_init(&gIter, e->voted);
while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) {
crm_err("voted: %s", key);
}
}
election_complete(e);
return TRUE;
} else {
crm_debug("Still waiting on %d non-votes (%d total)",
num_members - voted_size, num_members);
}
return FALSE;
}
#define loss_dampen 2 /* in seconds */
/* A_ELECTION_COUNT */
enum election_result
election_count_vote(election_t *e, xmlNode *vote, bool can_win)
{
int age = 0;
int election_id = -1;
int log_level = LOG_INFO;
gboolean use_born_on = FALSE;
gboolean done = FALSE;
gboolean we_lose = FALSE;
const char *op = NULL;
const char *from = NULL;
const char *reason = "unknown";
const char *election_owner = NULL;
crm_node_t *our_node = NULL, *your_node = NULL;
static int election_wins = 0;
xmlNode *novote = NULL;
time_t tm_now = time(NULL);
static time_t expires = 0;
static time_t last_election_loss = 0;
/* if the membership copy is NULL we REALLY shouldn't be voting
* the question is how we managed to get here.
*/
CRM_CHECK(vote != NULL, return election_error);
if(e == NULL) {
crm_info("Not voting in election: not initialized");
return election_lost;
} else if(crm_peer_cache == NULL) {
crm_info("Not voting in election: no peer cache");
return election_lost;
}
op = crm_element_value(vote, F_CRM_TASK);
from = crm_element_value(vote, F_CRM_HOST_FROM);
election_owner = crm_element_value(vote, F_CRM_ELECTION_OWNER);
crm_element_value_int(vote, F_CRM_ELECTION_ID, &election_id);
your_node = crm_get_peer(0, from);
our_node = crm_get_peer(0, e->uname);
if (e->voted == NULL) {
crm_debug("Created voted hash");
e->voted = g_hash_table_new_full(crm_str_hash, g_str_equal,
g_hash_destroy_str, g_hash_destroy_str);
}
if (is_heartbeat_cluster()) {
use_born_on = TRUE;
} else if (is_classic_ais_cluster()) {
use_born_on = TRUE;
}
if(can_win == FALSE) {
reason = "Not eligible";
we_lose = TRUE;
} else if (our_node == NULL || crm_is_peer_active(our_node) == FALSE) {
reason = "We are not part of the cluster";
log_level = LOG_ERR;
we_lose = TRUE;
} else if (election_id != e->count && crm_str_eq(our_node->uuid, election_owner, TRUE)) {
log_level = LOG_TRACE;
reason = "Superseded";
done = TRUE;
} else if (your_node == NULL || crm_is_peer_active(your_node) == FALSE) {
/* Possibly we cached the message in the FSA queue at a point that it wasn't */
reason = "Peer is not part of our cluster";
log_level = LOG_WARNING;
done = TRUE;
} else if (crm_str_eq(op, CRM_OP_NOVOTE, TRUE)) {
char *op_copy = strdup(op);
char *uname_copy = strdup(from);
CRM_ASSERT(crm_str_eq(our_node->uuid, election_owner, TRUE));
/* update the list of nodes that have voted */
g_hash_table_replace(e->voted, uname_copy, op_copy);
reason = "Recorded";
done = TRUE;
} else {
struct timeval your_age;
const char *your_version = crm_element_value(vote, F_CRM_VERSION);
int tv_sec = 0;
int tv_usec = 0;
crm_element_value_int(vote, F_CRM_ELECTION_AGE_S, &tv_sec);
crm_element_value_int(vote, F_CRM_ELECTION_AGE_US, &tv_usec);
your_age.tv_sec = tv_sec;
your_age.tv_usec = tv_usec;
age = crm_compare_age(your_age);
if (crm_str_eq(from, e->uname, TRUE)) {
char *op_copy = strdup(op);
char *uname_copy = strdup(from);
CRM_ASSERT(crm_str_eq(our_node->uuid, election_owner, TRUE));
/* update ourselves in the list of nodes that have voted */
g_hash_table_replace(e->voted, uname_copy, op_copy);
reason = "Recorded";
done = TRUE;
} else if (compare_version(your_version, CRM_FEATURE_SET) < 0) {
reason = "Version";
we_lose = TRUE;
} else if (compare_version(your_version, CRM_FEATURE_SET) > 0) {
reason = "Version";
} else if (age < 0) {
reason = "Uptime";
we_lose = TRUE;
} else if (age > 0) {
reason = "Uptime";
/* TODO: Check for y(our) born < 0 */
} else if (use_born_on && your_node->born < our_node->born) {
reason = "Born";
we_lose = TRUE;
} else if (use_born_on && your_node->born > our_node->born) {
reason = "Born";
} else if (e->uname == NULL) {
reason = "Unknown host name";
we_lose = TRUE;
} else if (strcasecmp(e->uname, from) > 0) {
reason = "Host name";
we_lose = TRUE;
} else {
reason = "Host name";
CRM_ASSERT(strcasecmp(e->uname, from) < 0);
/* can't happen...
* } else if(strcasecmp(e->uname, from) == 0) {
*
*/
}
}
if (expires < tm_now) {
election_wins = 0;
expires = tm_now + STORM_INTERVAL;
} else if (done == FALSE && we_lose == FALSE) {
int peers = 1 + g_hash_table_size(crm_peer_cache);
/* If every node has to vote down every other node, thats N*(N-1) total elections
* Allow some leway before _really_ complaining
*/
election_wins++;
if (election_wins > (peers * peers)) {
crm_warn("Election storm detected: %d elections in %d seconds", election_wins,
STORM_INTERVAL);
election_wins = 0;
expires = tm_now + STORM_INTERVAL;
crm_write_blackbox(0, NULL);
}
}
if (done) {
do_crm_log(log_level + 1, "Election %d (current: %d, owner: %s): Processed %s from %s (%s)",
election_id, e->count, election_owner, op, from, reason);
return e->state;
} else if (we_lose == FALSE) {
do_crm_log(log_level, "Election %d (owner: %s) pass: %s from %s (%s)",
election_id, election_owner, op, from, reason);
if (last_election_loss == 0
|| tm_now - last_election_loss > (time_t) loss_dampen) {
last_election_loss = 0;
election_timeout_stop(e);
/* Start a new election by voting down this, and other, peers */
e->state = election_start;
return e->state;
}
crm_info("Election %d ignore: We already lost an election less than %ds ago (%s)",
election_id, loss_dampen, ctime(&last_election_loss));
}
novote = create_request(CRM_OP_NOVOTE, NULL, from,
CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
do_crm_log(log_level, "Election %d (owner: %s) lost: %s from %s (%s)",
election_id, election_owner, op, from, reason);
election_timeout_stop(e);
crm_xml_add(novote, F_CRM_ELECTION_OWNER, election_owner);
crm_xml_add_int(novote, F_CRM_ELECTION_ID, election_id);
send_cluster_message(your_node, crm_msg_crmd, novote, TRUE);
free_xml(novote);
last_election_loss = tm_now;
e->state = election_lost;
return e->state;
}
diff --git a/lib/pengine/status.c b/lib/pengine/status.c
index 8dda5d4071..e2e166a0fa 100644
--- a/lib/pengine/status.c
+++ b/lib/pengine/status.c
@@ -1,311 +1,311 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <glib.h>
#include <crm/pengine/internal.h>
#include <unpack.h>
#define MEMCHECK_STAGE_0 0
#define check_and_exit(stage) cleanup_calculations(data_set); \
crm_mem_stats(NULL); \
crm_err("Exiting: stage %d", stage); \
crm_exit(pcmk_err_generic);
/*
* Unpack everything
* At the end you'll have:
* - A list of nodes
* - A list of resources (each with any dependencies on other resources)
* - A list of constraints between resources and nodes
* - A list of constraints between start/stop actions
* - A list of nodes that need to be stonith'd
* - A list of nodes that need to be shutdown
* - A list of the possible stop/start actions (without dependencies)
*/
gboolean
cluster_status(pe_working_set_t * data_set)
{
xmlNode *config = get_xpath_object("//"XML_CIB_TAG_CRMCONFIG, data_set->input, LOG_TRACE);
xmlNode *cib_nodes = get_xpath_object("//"XML_CIB_TAG_NODES, data_set->input, LOG_TRACE);
xmlNode *cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
xmlNode *cib_status = get_xpath_object("//"XML_CIB_TAG_STATUS, data_set->input, LOG_TRACE);
xmlNode *cib_tags = get_xpath_object("//"XML_CIB_TAG_TAGS, data_set->input, LOG_TRACE);
const char *value = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
crm_trace("Beginning unpack");
pe_dataset = data_set;
/* reset remaining global variables */
data_set->failed = create_xml_node(NULL, "failed-ops");
if (data_set->input == NULL) {
return FALSE;
}
if (data_set->now == NULL) {
data_set->now = crm_time_new(NULL);
}
if (data_set->dc_uuid == NULL
&& data_set->input != NULL
&& crm_element_value(data_set->input, XML_ATTR_DC_UUID) != NULL) {
/* this should always be present */
data_set->dc_uuid = crm_element_value_copy(data_set->input, XML_ATTR_DC_UUID);
}
clear_bit(data_set->flags, pe_flag_have_quorum);
if (crm_is_true(value)) {
set_bit(data_set->flags, pe_flag_have_quorum);
}
data_set->op_defaults = get_xpath_object("//"XML_CIB_TAG_OPCONFIG, data_set->input, LOG_TRACE);
data_set->rsc_defaults = get_xpath_object("//"XML_CIB_TAG_RSCCONFIG, data_set->input, LOG_TRACE);
unpack_config(config, data_set);
if (is_not_set(data_set->flags, pe_flag_quick_location)
&& is_not_set(data_set->flags, pe_flag_have_quorum)
&& data_set->no_quorum_policy != no_quorum_ignore) {
crm_warn("Fencing and resource management disabled due to lack of quorum");
}
unpack_nodes(cib_nodes, data_set);
if(is_not_set(data_set->flags, pe_flag_quick_location)) {
unpack_remote_nodes(cib_resources, data_set);
}
unpack_resources(cib_resources, data_set);
unpack_tags(cib_tags, data_set);
if(is_not_set(data_set->flags, pe_flag_quick_location)) {
unpack_status(cib_status, data_set);
}
set_bit(data_set->flags, pe_flag_have_status);
return TRUE;
}
static void
pe_free_resources(GListPtr resources)
{
resource_t *rsc = NULL;
GListPtr iterator = resources;
while (iterator != NULL) {
rsc = (resource_t *) iterator->data;
iterator = iterator->next;
rsc->fns->free(rsc);
}
if (resources != NULL) {
g_list_free(resources);
}
}
static void
pe_free_actions(GListPtr actions)
{
GListPtr iterator = actions;
while (iterator != NULL) {
pe_free_action(iterator->data);
iterator = iterator->next;
}
if (actions != NULL) {
g_list_free(actions);
}
}
static void
pe_free_nodes(GListPtr nodes)
{
GListPtr iterator = nodes;
while (iterator != NULL) {
node_t *node = (node_t *) iterator->data;
struct node_shared_s *details = node->details;
iterator = iterator->next;
crm_trace("deleting node");
print_node("delete", node, FALSE);
if (details != NULL) {
crm_trace("%s is being deleted", details->uname);
if (details->attrs != NULL) {
g_hash_table_destroy(details->attrs);
}
if (details->utilization != NULL) {
g_hash_table_destroy(details->utilization);
}
if (details->digest_cache != NULL) {
g_hash_table_destroy(details->digest_cache);
}
g_list_free(details->running_rsc);
g_list_free(details->allocated_rsc);
free(details);
}
free(node);
}
if (nodes != NULL) {
g_list_free(nodes);
}
}
void
cleanup_calculations(pe_working_set_t * data_set)
{
pe_dataset = NULL;
if (data_set == NULL) {
return;
}
clear_bit(data_set->flags, pe_flag_have_status);
if (data_set->config_hash != NULL) {
g_hash_table_destroy(data_set->config_hash);
}
if (data_set->singletons != NULL) {
g_hash_table_destroy(data_set->singletons);
}
if (data_set->tickets) {
g_hash_table_destroy(data_set->tickets);
}
if (data_set->template_rsc_sets) {
g_hash_table_destroy(data_set->template_rsc_sets);
}
if (data_set->tags) {
g_hash_table_destroy(data_set->tags);
}
free(data_set->dc_uuid);
crm_trace("deleting resources");
pe_free_resources(data_set->resources);
crm_trace("deleting actions");
pe_free_actions(data_set->actions);
crm_trace("deleting nodes");
pe_free_nodes(data_set->nodes);
free_xml(data_set->graph);
crm_time_free(data_set->now);
free_xml(data_set->input);
free_xml(data_set->failed);
set_working_set_defaults(data_set);
CRM_CHECK(data_set->ordering_constraints == NULL,;
);
CRM_CHECK(data_set->placement_constraints == NULL,;
);
}
void
set_working_set_defaults(pe_working_set_t * data_set)
{
pe_dataset = data_set;
memset(data_set, 0, sizeof(pe_working_set_t));
data_set->dc_uuid = NULL;
data_set->order_id = 1;
data_set->action_id = 1;
data_set->no_quorum_policy = no_quorum_freeze;
data_set->flags = 0x0ULL;
set_bit(data_set->flags, pe_flag_stop_rsc_orphans);
set_bit(data_set->flags, pe_flag_symmetric_cluster);
set_bit(data_set->flags, pe_flag_is_managed_default);
set_bit(data_set->flags, pe_flag_stop_action_orphans);
}
resource_t *
pe_find_resource(GListPtr rsc_list, const char *id)
{
GListPtr rIter = NULL;
for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
resource_t *parent = rIter->data;
resource_t *match =
parent->fns->find_rsc(parent, id, NULL, pe_find_renamed | pe_find_current);
if (match != NULL) {
return match;
}
}
crm_trace("No match for %s", id);
return NULL;
}
node_t *
pe_find_node_any(GListPtr nodes, const char *id, const char *uname)
{
node_t *match = pe_find_node_id(nodes, id);
if (match) {
return match;
}
- crm_trace("Looking up %s via it's uname instead", uname);
+ crm_trace("Looking up %s via its uname instead", uname);
return pe_find_node(nodes, uname);
}
node_t *
pe_find_node_id(GListPtr nodes, const char *id)
{
GListPtr gIter = nodes;
for (; gIter != NULL; gIter = gIter->next) {
node_t *node = (node_t *) gIter->data;
if (node && safe_str_eq(node->details->id, id)) {
return node;
}
}
/* error */
return NULL;
}
node_t *
pe_find_node(GListPtr nodes, const char *uname)
{
GListPtr gIter = nodes;
for (; gIter != NULL; gIter = gIter->next) {
node_t *node = (node_t *) gIter->data;
if (node && safe_str_eq(node->details->uname, uname)) {
return node;
}
}
/* error */
return NULL;
}
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 40b47d8c22..42e8b071e0 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,1693 +1,1693 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_resource.h>
bool do_trace = FALSE;
bool do_force = FALSE;
int crmd_replies_needed = 1; /* The welcome message */
const char *attr_set_type = XML_TAG_ATTR_SETS;
static int
do_find_resource(const char *rsc, resource_t * the_rsc, pe_working_set_t * data_set)
{
int found = 0;
GListPtr lpc = NULL;
for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
node_t *node = (node_t *) lpc->data;
crm_trace("resource %s is running on: %s", rsc, node->details->uname);
if (BE_QUIET) {
fprintf(stdout, "%s\n", node->details->uname);
} else {
const char *state = "";
if (the_rsc->variant < pe_clone && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
state = "Master";
}
fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
}
found++;
}
if (BE_QUIET == FALSE && found == 0) {
fprintf(stderr, "resource %s is NOT running\n", rsc);
}
return found;
}
int
cli_resource_search(const char *rsc, pe_working_set_t * data_set)
{
int found = 0;
resource_t *the_rsc = NULL;
resource_t *parent = NULL;
if (the_rsc == NULL) {
the_rsc = pe_find_resource(data_set->resources, rsc);
}
if (the_rsc == NULL) {
return -ENXIO;
}
if (the_rsc->variant >= pe_clone) {
GListPtr gIter = the_rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
found += do_find_resource(rsc, gIter->data, data_set);
}
/* The anonymous clone children's common ID is supplied */
} else if ((parent = uber_parent(the_rsc)) != NULL
&& parent->variant >= pe_clone
&& is_not_set(the_rsc->flags, pe_rsc_unique)
&& the_rsc->clone_name
&& safe_str_eq(rsc, the_rsc->clone_name)
&& safe_str_neq(rsc, the_rsc->id)) {
GListPtr gIter = parent->children;
for (; gIter != NULL; gIter = gIter->next) {
found += do_find_resource(rsc, gIter->data, data_set);
}
} else {
found += do_find_resource(rsc, the_rsc, data_set);
}
return found;
}
resource_t *
find_rsc_or_clone(const char *rsc, pe_working_set_t * data_set)
{
resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
if (the_rsc == NULL) {
char *as_clone = crm_concat(rsc, "0", ':');
the_rsc = pe_find_resource(data_set->resources, as_clone);
free(as_clone);
}
return the_rsc;
}
static int
find_resource_attr(cib_t * the_cib, const char *attr, const char *rsc, const char *set_type,
const char *set_name, const char *attr_id, const char *attr_name, char **value)
{
int offset = 0;
static int xpath_max = 1024;
int rc = pcmk_ok;
xmlNode *xml_search = NULL;
char *xpath_string = NULL;
if(value) {
*value = NULL;
}
if(the_cib == NULL) {
return -ENOTCONN;
}
xpath_string = calloc(1, xpath_max);
offset +=
snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path("resources"));
offset += snprintf(xpath_string + offset, xpath_max - offset, "//*[@id=\"%s\"]", rsc);
if (set_type) {
offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s", set_type);
if (set_name) {
offset += snprintf(xpath_string + offset, xpath_max - offset, "[@id=\"%s\"]", set_name);
}
}
offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
if (attr_id) {
offset += snprintf(xpath_string + offset, xpath_max - offset, "@id=\"%s\"", attr_id);
}
if (attr_name) {
if (attr_id) {
offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
}
offset += snprintf(xpath_string + offset, xpath_max - offset, "@name=\"%s\"", attr_name);
}
offset += snprintf(xpath_string + offset, xpath_max - offset, "]");
CRM_LOG_ASSERT(offset > 0);
rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
cib_sync_call | cib_scope_local | cib_xpath);
if (rc != pcmk_ok) {
goto bail;
}
crm_log_xml_debug(xml_search, "Match");
if (xml_has_children(xml_search)) {
xmlNode *child = NULL;
rc = -EINVAL;
printf("Multiple attributes match name=%s\n", attr_name);
for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
printf(" Value: %s \t(id=%s)\n",
crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
}
} else if(value) {
const char *tmp = crm_element_value(xml_search, attr);
if (tmp) {
*value = strdup(tmp);
}
}
bail:
free(xpath_string);
free_xml(xml_search);
return rc;
}
static resource_t *
find_matching_attr_resource(resource_t * rsc, const char * rsc_id, const char * attr_set, const char * attr_id,
const char * attr_name, cib_t * cib, const char * cmd)
{
int rc = pcmk_ok;
char *lookup_id = NULL;
char *local_attr_id = NULL;
if(do_force == TRUE) {
return rsc;
} else if(rsc->parent) {
switch(rsc->parent->variant) {
case pe_group:
if (BE_QUIET == FALSE) {
printf("Performing %s of '%s' for '%s' will not apply to its peers in '%s'\n", cmd, attr_name, rsc_id, rsc->parent->id);
}
break;
case pe_master:
case pe_clone:
rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
free(local_attr_id);
if(rc != pcmk_ok) {
rsc = rsc->parent;
if (BE_QUIET == FALSE) {
printf("Performing %s of '%s' on '%s', the parent of '%s'\n", cmd, attr_name, rsc->id, rsc_id);
}
}
break;
default:
break;
}
} else if (rsc->parent && BE_QUIET == FALSE) {
printf("Forcing %s of '%s' for '%s' instead of '%s'\n", cmd, attr_name, rsc_id, rsc->parent->id);
} else if(rsc->parent == NULL && rsc->children) {
resource_t *child = rsc->children->data;
if(child->variant == pe_native) {
lookup_id = clone_strip(child->id); /* Could be a cloned group! */
rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
if(rc == pcmk_ok) {
rsc = child;
if (BE_QUIET == FALSE) {
printf("A value for '%s' already exists in child '%s', performing %s on that instead of '%s'\n", attr_name, lookup_id, cmd, rsc_id);
}
}
free(local_attr_id);
free(lookup_id);
}
}
return rsc;
}
int
cli_resource_update_attribute(const char *rsc_id, const char *attr_set, const char *attr_id,
const char *attr_name, const char *attr_value, bool recursive,
cib_t * cib, pe_working_set_t * data_set)
{
int rc = pcmk_ok;
static bool need_init = TRUE;
char *lookup_id = NULL;
char *local_attr_id = NULL;
char *local_attr_set = NULL;
xmlNode *xml_top = NULL;
xmlNode *xml_obj = NULL;
bool use_attributes_tag = FALSE;
resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
if (rsc == NULL) {
return -ENXIO;
}
if(attr_id == NULL
&& do_force == FALSE
&& pcmk_ok != find_resource_attr(
cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL)) {
printf("\n");
}
if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
if (do_force == FALSE) {
rc = find_resource_attr(cib, XML_ATTR_ID, uber_parent(rsc)->id,
XML_TAG_META_SETS, attr_set, attr_id,
attr_name, &local_attr_id);
if (rc == pcmk_ok && BE_QUIET == FALSE) {
printf("WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)\n",
uber_parent(rsc)->id, attr_name, local_attr_id);
printf(" Delete '%s' first or use --force to override\n", local_attr_id);
}
free(local_attr_id);
if (rc == pcmk_ok) {
return -ENOTUNIQ;
}
}
} else {
rsc = find_matching_attr_resource(rsc, rsc_id, attr_set, attr_id, attr_name, cib, "update");
}
lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
&local_attr_id);
if (rc == pcmk_ok) {
crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
attr_id = local_attr_id;
} else if (rc != -ENXIO) {
free(lookup_id);
free(local_attr_id);
return rc;
} else {
const char *value = NULL;
xmlNode *cib_top = NULL;
const char *tag = crm_element_name(rsc->xml);
cib->cmds->query(cib, "/cib", &cib_top,
cib_sync_call | cib_scope_local | cib_xpath | cib_no_children);
value = crm_element_value(cib_top, "ignore_dtd");
if (value != NULL) {
use_attributes_tag = TRUE;
} else {
value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
if (crm_ends_with(value, "-0.6")) {
use_attributes_tag = TRUE;
}
}
free_xml(cib_top);
if (attr_set == NULL) {
local_attr_set = crm_concat(lookup_id, attr_set_type, '-');
attr_set = local_attr_set;
}
if (attr_id == NULL) {
local_attr_id = crm_concat(attr_set, attr_name, '-');
attr_id = local_attr_id;
}
if (use_attributes_tag && safe_str_eq(tag, XML_CIB_TAG_MASTER)) {
tag = "master_slave"; /* use the old name */
}
xml_top = create_xml_node(NULL, tag);
crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
xml_obj = create_xml_node(xml_top, attr_set_type);
crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
if (use_attributes_tag) {
xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
}
}
xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
if (xml_top == NULL) {
xml_top = xml_obj;
}
crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);
crm_log_xml_debug(xml_top, "Update");
rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
if (rc == pcmk_ok && BE_QUIET == FALSE) {
printf("Set '%s' option: id=%s%s%s%s%s=%s\n", lookup_id, local_attr_id,
attr_set ? " set=" : "", attr_set ? attr_set : "",
attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
}
free_xml(xml_top);
free(lookup_id);
free(local_attr_id);
free(local_attr_set);
if(recursive && safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
GListPtr lpc = NULL;
if(need_init) {
xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
need_init = FALSE;
unpack_constraints(cib_constraints, data_set);
for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
resource_t *r = (resource_t *) lpc->data;
clear_bit(r->flags, pe_rsc_allocating);
}
}
crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
set_bit(rsc->flags, pe_rsc_allocating);
for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
resource_t *peer = cons->rsc_lh;
crm_debug("Checking %s %d", cons->id, cons->score);
if (cons->score > 0 && is_not_set(peer->flags, pe_rsc_allocating)) {
/* Don't get into colocation loops */
crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
cli_resource_update_attribute(peer->id, NULL, NULL, attr_name, attr_value, recursive, cib, data_set);
}
}
}
return rc;
}
int
cli_resource_delete_attribute(const char *rsc_id, const char *attr_set, const char *attr_id,
const char *attr_name, cib_t * cib, pe_working_set_t * data_set)
{
xmlNode *xml_obj = NULL;
int rc = pcmk_ok;
char *lookup_id = NULL;
char *local_attr_id = NULL;
resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
if (rsc == NULL) {
return -ENXIO;
}
if(attr_id == NULL
&& do_force == FALSE
&& find_resource_attr(
cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) != pcmk_ok) {
printf("\n");
}
if(safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
rsc = find_matching_attr_resource(rsc, rsc_id, attr_set, attr_id, attr_name, cib, "delete");
}
lookup_id = clone_strip(rsc->id);
rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
&local_attr_id);
if (rc == -ENXIO) {
free(lookup_id);
return pcmk_ok;
} else if (rc != pcmk_ok) {
free(lookup_id);
return rc;
}
if (attr_id == NULL) {
attr_id = local_attr_id;
}
xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
crm_log_xml_debug(xml_obj, "Delete");
CRM_ASSERT(cib);
rc = cib->cmds->delete(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
if (rc == pcmk_ok && BE_QUIET == FALSE) {
printf("Deleted '%s' option: id=%s%s%s%s%s\n", lookup_id, local_attr_id,
attr_set ? " set=" : "", attr_set ? attr_set : "",
attr_name ? " name=" : "", attr_name ? attr_name : "");
}
free(lookup_id);
free_xml(xml_obj);
free(local_attr_id);
return rc;
}
static int
send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
const char *host_uname, const char *rsc_id,
bool only_failed, pe_working_set_t * data_set)
{
char *our_pid = NULL;
char *key = NULL;
int rc = -ECOMM;
xmlNode *cmd = NULL;
xmlNode *xml_rsc = NULL;
const char *value = NULL;
const char *router_node = host_uname;
xmlNode *params = NULL;
xmlNode *msg_data = NULL;
resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
if (rsc == NULL) {
CMD_ERR("Resource %s not found", rsc_id);
return -ENXIO;
} else if (rsc->variant != pe_native) {
CMD_ERR("We can only process primitive resources, not %s", rsc_id);
return -EINVAL;
} else if (host_uname == NULL) {
CMD_ERR("Please supply a hostname with -H");
return -EINVAL;
} else {
node_t *node = pe_find_node(data_set->nodes, host_uname);
if (node && is_remote_node(node)) {
if (node->details->remote_rsc == NULL || node->details->remote_rsc->running_on == NULL) {
CMD_ERR("No lrmd connection detected to remote node %s", host_uname);
return -ENXIO;
}
node = node->details->remote_rsc->running_on->data;
router_node = node->details->uname;
}
}
key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
free(key);
crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
if (safe_str_neq(router_node, host_uname)) {
crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
}
xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
if (rsc->clone_name) {
crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);
} else {
crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
}
value = crm_element_value(rsc->xml, XML_ATTR_TYPE);
crm_xml_add(xml_rsc, XML_ATTR_TYPE, value);
if (value == NULL) {
CMD_ERR("%s has no type! Aborting...", rsc_id);
return -ENXIO;
}
value = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, value);
if (value == NULL) {
CMD_ERR("%s has no class! Aborting...", rsc_id);
return -ENXIO;
}
value = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, value);
params = create_xml_node(msg_data, XML_TAG_ATTRS);
crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
crm_xml_add(params, key, "60000"); /* 1 minute */
free(key);
our_pid = calloc(1, 11);
if (our_pid != NULL) {
snprintf(our_pid, 10, "%d", getpid());
our_pid[10] = '\0';
}
cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid);
/* crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
free_xml(msg_data);
if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
rc = 0;
} else {
CMD_ERR("Could not send %s op to the crmd", op);
rc = -ENOTCONN;
}
free_xml(cmd);
return rc;
}
static int
cli_delete_attr(cib_t * cib_conn, const char * host_uname, const char * attr_name,
pe_working_set_t * data_set)
{
node_t *node = pe_find_node(data_set->nodes, host_uname);
int attr_options = attrd_opt_none;
if (node && is_remote_node(node)) {
#if HAVE_ATOMIC_ATTRD
set_bit(attr_options, attrd_opt_remote);
#else
/* Talk directly to cib for remote nodes if it's legacy attrd */
return delete_attr_delegate(cib_conn, cib_sync_call, XML_CIB_TAG_STATUS, node->details->id, NULL, NULL,
NULL, attr_name, NULL, FALSE, NULL);
#endif
}
return attrd_update_delegate(NULL, 'D', host_uname, attr_name, NULL,
XML_CIB_TAG_STATUS, NULL, NULL, NULL,
attr_options);
}
int
cli_resource_delete(cib_t *cib_conn, crm_ipc_t * crmd_channel, const char *host_uname,
resource_t * rsc, pe_working_set_t * data_set)
{
int rc = pcmk_ok;
node_t *node = NULL;
if (rsc == NULL) {
return -ENXIO;
} else if (rsc->children) {
GListPtr lpc = NULL;
for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
resource_t *child = (resource_t *) lpc->data;
rc = cli_resource_delete(cib_conn, crmd_channel, host_uname, child, data_set);
if(rc != pcmk_ok
|| (rsc->variant >= pe_clone && is_not_set(rsc->flags, pe_rsc_unique))) {
return rc;
}
}
return pcmk_ok;
} else if (host_uname == NULL) {
GListPtr lpc = NULL;
for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) {
node = (node_t *) lpc->data;
if (node->details->online) {
cli_resource_delete(cib_conn, crmd_channel, node->details->uname, rsc, data_set);
}
}
return pcmk_ok;
}
node = pe_find_node(data_set->nodes, host_uname);
if (node && node->details->rsc_discovery_enabled) {
printf("Cleaning up %s on %s", rsc->id, host_uname);
rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
} else {
printf("Resource discovery disabled on %s. Unable to delete lrm state.\n", host_uname);
rc = -EOPNOTSUPP;
}
if (rc == pcmk_ok) {
char *attr_name = NULL;
if(node && node->details->remote_rsc == NULL && node->details->rsc_discovery_enabled) {
crmd_replies_needed++;
}
if(is_not_set(rsc->flags, pe_rsc_unique)) {
char *id = clone_strip(rsc->id);
attr_name = crm_strdup_printf("fail-count-%s", id);
free(id);
} else if (rsc->clone_name) {
attr_name = crm_strdup_printf("fail-count-%s", rsc->clone_name);
} else {
attr_name = crm_strdup_printf("fail-count-%s", rsc->id);
}
printf(", removing %s\n", attr_name);
rc = cli_delete_attr(cib_conn, host_uname, attr_name, data_set);
free(attr_name);
} else if(rc != -EOPNOTSUPP) {
printf(" - FAILED\n");
}
return rc;
}
void
cli_resource_check(cib_t * cib_conn, resource_t *rsc)
{
int need_nl = 0;
char *role_s = NULL;
char *managed = NULL;
resource_t *parent = uber_parent(rsc);
find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
if(role_s) {
enum rsc_role_e role = text2role(role_s);
if(role == RSC_ROLE_UNKNOWN) {
// Treated as if unset
} else if(role == RSC_ROLE_STOPPED) {
printf("\n * The configuration specifies that '%s' should remain stopped\n", parent->id);
need_nl++;
} else if(parent->variant > pe_clone && role == RSC_ROLE_SLAVE) {
printf("\n * The configuration specifies that '%s' should not be promoted\n", parent->id);
need_nl++;
}
}
if(managed && crm_is_true(managed) == FALSE) {
printf("%s * The configuration prevents the cluster from stopping or starting '%s' (unmanaged)\n", need_nl == 0?"\n":"", parent->id);
need_nl++;
}
if(need_nl) {
printf("\n");
}
}
int
cli_resource_fail(crm_ipc_t * crmd_channel, const char *host_uname,
const char *rsc_id, pe_working_set_t * data_set)
{
crm_warn("Failing: %s", rsc_id);
return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
}
static GHashTable *
generate_resource_params(resource_t * rsc, pe_working_set_t * data_set)
{
GHashTable *params = NULL;
GHashTable *meta = NULL;
GHashTable *combined = NULL;
GHashTableIter iter;
if (!rsc) {
crm_err("Resource does not exist in config");
return NULL;
}
params =
g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
meta = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
combined =
g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);
if (params) {
char *key = NULL;
char *value = NULL;
g_hash_table_iter_init(&iter, params);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
g_hash_table_insert(combined, strdup(key), strdup(value));
}
g_hash_table_destroy(params);
}
if (meta) {
char *key = NULL;
char *value = NULL;
g_hash_table_iter_init(&iter, meta);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
char *crm_name = crm_meta_name(key);
g_hash_table_insert(combined, crm_name, strdup(value));
}
g_hash_table_destroy(meta);
}
return combined;
}
static bool resource_is_running_on(resource_t *rsc, const char *host)
{
bool found = TRUE;
GListPtr hIter = NULL;
GListPtr hosts = NULL;
if(rsc == NULL) {
return FALSE;
}
rsc->fns->location(rsc, &hosts, TRUE);
for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
pe_node_t *node = (pe_node_t *) hIter->data;
if(strcmp(host, node->details->uname) == 0) {
crm_trace("Resource %s is running on %s\n", rsc->id, host);
goto done;
} else if(strcmp(host, node->details->id) == 0) {
crm_trace("Resource %s is running on %s\n", rsc->id, host);
goto done;
}
}
if(host != NULL) {
crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
found = FALSE;
} else if(host == NULL && hosts == NULL) {
crm_trace("Resource %s is not running\n", rsc->id);
found = FALSE;
}
done:
g_list_free(hosts);
return found;
}
/*!
* \internal
* \brief Create a list of all resources active on host from a given list
*
* \param[in] host Name of host to check whether resources are active
* \param[in] rsc_list List of resources to check
*
* \return New list of resources from list that are active on host
*/
static GList *
get_active_resources(const char *host, GList *rsc_list)
{
GList *rIter = NULL;
GList *active = NULL;
for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
resource_t *rsc = (resource_t *) rIter->data;
/* Expand groups to their members, because if we're restarting a member
* other than the first, we can't otherwise tell which resources are
* stopping and starting.
*/
if (rsc->variant == pe_group) {
active = g_list_concat(active,
get_active_resources(host, rsc->children));
} else if (resource_is_running_on(rsc, host)) {
active = g_list_append(active, strdup(rsc->id));
}
}
return active;
}
static GList *subtract_lists(GList *from, GList *items)
{
GList *item = NULL;
GList *result = g_list_copy(from);
for (item = items; item != NULL; item = item->next) {
GList *candidate = NULL;
for (candidate = from; candidate != NULL; candidate = candidate->next) {
crm_info("Comparing %s with %s", candidate->data, item->data);
if(strcmp(candidate->data, item->data) == 0) {
result = g_list_remove(result, candidate->data);
break;
}
}
}
return result;
}
static void dump_list(GList *items, const char *tag)
{
int lpc = 0;
GList *item = NULL;
for (item = items; item != NULL; item = item->next) {
crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
lpc++;
}
}
static void display_list(GList *items, const char *tag)
{
GList *item = NULL;
for (item = items; item != NULL; item = item->next) {
fprintf(stdout, "%s%s\n", tag, (const char *)item->data);
}
}
/*!
* \internal
* \brief Upgrade XML to latest schema version and use it as working set input
*
* This also updates the working set timestamp to the current time.
*
* \param[in] data_set Working set instance to update
* \param[in] xml XML to use as input
*
* \return pcmk_ok on success, -ENOKEY if unable to upgrade XML
* \note On success, caller is responsible for freeing memory allocated for
* data_set->now.
* \todo This follows the example of other callers of cli_config_update()
* and returns -ENOKEY ("Required key not available") if that fails,
* but perhaps -pcmk_err_schema_validation would be better in that case.
*/
int
update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
{
if (cli_config_update(xml, NULL, FALSE) == FALSE) {
return -ENOKEY;
}
data_set->input = *xml;
data_set->now = crm_time_new(NULL);
return pcmk_ok;
}
/*!
* \internal
* \brief Update a working set's XML input based on a CIB query
*
* \param[in] data_set Data set instance to initialize
* \param[in] cib Connection to the CIB
*
* \return pcmk_ok on success, -errno on failure
* \note On success, caller is responsible for freeing memory allocated for
* data_set->input and data_set->now.
*/
static int
update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
{
xmlNode *cib_xml_copy = NULL;
int rc;
rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
if (rc != pcmk_ok) {
fprintf(stderr, "Could not obtain the current CIB: %s (%d)\n", pcmk_strerror(rc), rc);
return rc;
}
rc = update_working_set_xml(data_set, &cib_xml_copy);
if (rc != pcmk_ok) {
fprintf(stderr, "Could not upgrade the current CIB XML\n");
free_xml(cib_xml_copy);
return rc;
}
return pcmk_ok;
}
static int
update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
{
char *pid = NULL;
char *shadow_file = NULL;
cib_t *shadow_cib = NULL;
int rc;
cleanup_alloc_calculations(data_set);
rc = update_working_set_from_cib(data_set, cib);
if (rc != pcmk_ok) {
return rc;
}
if(simulate) {
pid = crm_itoa(getpid());
shadow_cib = cib_shadow_new(pid);
shadow_file = get_shadow_file(pid);
if (shadow_cib == NULL) {
fprintf(stderr, "Could not create shadow cib: '%s'\n", pid);
rc = -ENXIO;
goto cleanup;
}
rc = write_xml_file(data_set->input, shadow_file, FALSE);
if (rc < 0) {
fprintf(stderr, "Could not populate shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
goto cleanup;
}
rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
if(rc != pcmk_ok) {
fprintf(stderr, "Could not connect to shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
goto cleanup;
}
do_calculations(data_set, data_set->input, NULL);
run_simulation(data_set, shadow_cib, NULL, TRUE);
rc = update_dataset(shadow_cib, data_set, FALSE);
} else {
cluster_status(data_set);
}
cleanup:
/* Do not free data_set->input here, we need rsc->xml to be valid later on */
cib_delete(shadow_cib);
free(pid);
if(shadow_file) {
unlink(shadow_file);
free(shadow_file);
}
return rc;
}
static int
max_delay_for_resource(pe_working_set_t * data_set, resource_t *rsc)
{
int delay = 0;
int max_delay = 0;
if(rsc && rsc->children) {
GList *iter = NULL;
for(iter = rsc->children; iter; iter = iter->next) {
resource_t *child = (resource_t *)iter->data;
delay = max_delay_for_resource(data_set, child);
if(delay > max_delay) {
double seconds = delay / 1000.0;
crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
max_delay = delay;
}
}
} else if(rsc) {
char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
max_delay = crm_int_helper(value, NULL);
pe_free_action(stop);
}
return max_delay;
}
static int
max_delay_in(pe_working_set_t * data_set, GList *resources)
{
int max_delay = 0;
GList *item = NULL;
for (item = resources; item != NULL; item = item->next) {
int delay = 0;
resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
delay = max_delay_for_resource(data_set, rsc);
if(delay > max_delay) {
double seconds = delay / 1000.0;
crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
max_delay = delay;
}
}
return 5 + (max_delay / 1000);
}
#define waiting_for_starts(d, r, h) ((g_list_length(d) > 0) || \
(resource_is_running_on((r), (h)) == FALSE))
/*!
* \internal
* \brief Restart a resource (on a particular host if requested).
*
* \param[in] rsc The resource to restart
* \param[in] host The host to restart the resource on (or NULL for all)
* \param[in] timeout_ms Consider failed if actions do not complete in this time
* (specified in milliseconds, but a two-second
* granularity is actually used; if 0, a timeout will be
* calculated based on the resource timeout)
* \param[in] cib Connection to the CIB for modifying/checking resource
*
* \return pcmk_ok on success, -errno on failure (exits on certain failures)
*/
int
cli_resource_restart(resource_t * rsc, const char *host, int timeout_ms, cib_t * cib)
{
int rc = 0;
int lpc = 0;
int before = 0;
int step_timeout_s = 0;
int sleep_interval = 2;
int timeout = timeout_ms / 1000;
bool is_clone = FALSE;
char *rsc_id = NULL;
char *orig_target_role = NULL;
GList *list_delta = NULL;
GList *target_active = NULL;
GList *current_active = NULL;
GList *restart_target_active = NULL;
pe_working_set_t data_set;
if(resource_is_running_on(rsc, host) == FALSE) {
const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
if(host) {
printf("%s is not running on %s and so cannot be restarted\n", id, host);
} else {
printf("%s is not running anywhere and so cannot be restarted\n", id);
}
return -ENXIO;
}
/* We might set the target-role meta-attribute */
attr_set_type = XML_TAG_META_SETS;
rsc_id = strdup(rsc->id);
if(rsc->variant > pe_group) {
is_clone = TRUE;
}
/*
grab full cib
determine originally active resources
disable or ban
poll cib and watch for affected resources to get stopped
without --timeout, calculate the stop timeout for each step and wait for that
if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
if everything stopped, re-enable or un-ban
poll cib and watch for affected resources to get started
without --timeout, calculate the start timeout for each step and wait for that
if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
report success
Optimizations:
- use constraints to determine ordered list of affected resources
- Allow a --no-deps option (aka. --force-restart)
*/
set_working_set_defaults(&data_set);
rc = update_dataset(cib, &data_set, FALSE);
if(rc != pcmk_ok) {
fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
free(rsc_id);
return rc;
}
restart_target_active = get_active_resources(host, data_set.resources);
current_active = get_active_resources(host, data_set.resources);
dump_list(current_active, "Origin");
if(is_clone && host) {
/* Stop the clone instance by banning it from the host */
BE_QUIET = TRUE;
rc = cli_resource_ban(rsc_id, host, NULL, cib);
} else {
/* Stop the resource by setting target-role to Stopped.
* Remember any existing target-role so we can restore it later
* (though it only makes any difference if it's Slave).
*/
char *lookup_id = clone_strip(rsc->id);
find_resource_attr(cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
free(lookup_id);
rc = cli_resource_update_attribute(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, RSC_STOPPED, FALSE, cib, &data_set);
}
if(rc != pcmk_ok) {
fprintf(stderr, "Could not set target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
if (current_active) {
g_list_free_full(current_active, free);
}
if (restart_target_active) {
g_list_free_full(restart_target_active, free);
}
free(rsc_id);
return crm_exit(rc);
}
rc = update_dataset(cib, &data_set, TRUE);
if(rc != pcmk_ok) {
fprintf(stderr, "Could not determine which resources would be stopped\n");
goto failure;
}
target_active = get_active_resources(host, data_set.resources);
dump_list(target_active, "Target");
list_delta = subtract_lists(current_active, target_active);
fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
display_list(list_delta, " * ");
step_timeout_s = timeout / sleep_interval;
while(g_list_length(list_delta) > 0) {
before = g_list_length(list_delta);
if(timeout_ms == 0) {
step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
}
/* We probably don't need the entire step timeout */
for(lpc = 0; lpc < step_timeout_s && g_list_length(list_delta) > 0; lpc++) {
sleep(sleep_interval);
if(timeout) {
timeout -= sleep_interval;
crm_trace("%ds remaining", timeout);
}
rc = update_dataset(cib, &data_set, FALSE);
if(rc != pcmk_ok) {
fprintf(stderr, "Could not determine which resources were stopped\n");
goto failure;
}
if (current_active) {
g_list_free_full(current_active, free);
}
current_active = get_active_resources(host, data_set.resources);
g_list_free(list_delta);
list_delta = subtract_lists(current_active, target_active);
dump_list(current_active, "Current");
dump_list(list_delta, "Delta");
}
crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
if(before == g_list_length(list_delta)) {
/* aborted during stop phase, print the contents of list_delta */
fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
display_list(list_delta, " * ");
rc = -ETIME;
goto failure;
}
}
if(is_clone && host) {
rc = cli_resource_clear(rsc_id, host, NULL, cib);
} else if (orig_target_role) {
rc = cli_resource_update_attribute(rsc_id, NULL, NULL,
XML_RSC_ATTR_TARGET_ROLE,
orig_target_role, FALSE, cib,
&data_set);
free(orig_target_role);
orig_target_role = NULL;
} else {
rc = cli_resource_delete_attribute(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
}
if(rc != pcmk_ok) {
fprintf(stderr, "Could not unset target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
free(rsc_id);
return crm_exit(rc);
}
if (target_active) {
g_list_free_full(target_active, free);
}
target_active = restart_target_active;
if (list_delta) {
g_list_free(list_delta);
}
list_delta = subtract_lists(target_active, current_active);
fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
display_list(list_delta, " * ");
step_timeout_s = timeout / sleep_interval;
while (waiting_for_starts(list_delta, rsc, host)) {
before = g_list_length(list_delta);
if(timeout_ms == 0) {
step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
}
/* We probably don't need the entire step timeout */
for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
sleep(sleep_interval);
if(timeout) {
timeout -= sleep_interval;
crm_trace("%ds remaining", timeout);
}
rc = update_dataset(cib, &data_set, FALSE);
if(rc != pcmk_ok) {
fprintf(stderr, "Could not determine which resources were started\n");
goto failure;
}
if (current_active) {
g_list_free_full(current_active, free);
}
/* It's OK if dependent resources moved to a different node,
* so we check active resources on all nodes.
*/
current_active = get_active_resources(NULL, data_set.resources);
g_list_free(list_delta);
list_delta = subtract_lists(target_active, current_active);
dump_list(current_active, "Current");
dump_list(list_delta, "Delta");
}
if(before == g_list_length(list_delta)) {
/* aborted during start phase, print the contents of list_delta */
fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
display_list(list_delta, " * ");
rc = -ETIME;
goto failure;
}
}
rc = pcmk_ok;
goto done;
failure:
if(is_clone && host) {
cli_resource_clear(rsc_id, host, NULL, cib);
} else if (orig_target_role) {
cli_resource_update_attribute(rsc_id, NULL, NULL,
XML_RSC_ATTR_TARGET_ROLE,
orig_target_role, FALSE, cib, &data_set);
free(orig_target_role);
} else {
cli_resource_delete_attribute(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
}
done:
if (list_delta) {
g_list_free(list_delta);
}
if (current_active) {
g_list_free_full(current_active, free);
}
if (target_active && (target_active != restart_target_active)) {
g_list_free_full(target_active, free);
}
if (restart_target_active) {
g_list_free_full(restart_target_active, free);
}
cleanup_alloc_calculations(&data_set);
free(rsc_id);
return rc;
}
#define action_is_pending(action) \
((is_set((action)->flags, pe_action_optional) == FALSE) \
&& (is_set((action)->flags, pe_action_runnable) == TRUE) \
&& (is_set((action)->flags, pe_action_pseudo) == FALSE))
/*!
* \internal
* \brief Return TRUE if any actions in a list are pending
*
* \param[in] actions List of actions to check
*
* \return TRUE if any actions in the list are pending, FALSE otherwise
*/
static bool
actions_are_pending(GListPtr actions)
{
GListPtr action;
for (action = actions; action != NULL; action = action->next) {
if (action_is_pending((action_t *) action->data)) {
return TRUE;
}
}
return FALSE;
}
/*!
* \internal
* \brief Print pending actions to stderr
*
* \param[in] actions List of actions to check
*
* \return void
*/
static void
print_pending_actions(GListPtr actions)
{
GListPtr action;
fprintf(stderr, "Pending actions:\n");
for (action = actions; action != NULL; action = action->next) {
action_t *a = (action_t *) action->data;
if (action_is_pending(a)) {
fprintf(stderr, "\tAction %d: %s", a->id, a->uuid);
if (a->node) {
fprintf(stderr, "\ton %s", a->node->details->uname);
}
fprintf(stderr, "\n");
}
}
}
/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
/* For --wait, how long to sleep between cluster state checks */
#define WAIT_SLEEP_S (2)
/*!
* \internal
* \brief Wait until all pending cluster actions are complete
*
* This waits until either the CIB's transition graph is idle or a timeout is
* reached.
*
* \param[in] timeout_ms Consider failed if actions do not complete in this time
* (specified in milliseconds, but one-second granularity
* is actually used; if 0, a default will be used)
* \param[in] cib Connection to the CIB
*
* \return pcmk_ok on success, -errno on failure
*/
int
wait_till_stable(int timeout_ms, cib_t * cib)
{
pe_working_set_t data_set;
int rc = -1;
int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
time_t expire_time = time(NULL) + timeout_s;
time_t time_diff;
set_working_set_defaults(&data_set);
do {
/* Abort if timeout is reached */
time_diff = expire_time - time(NULL);
if (time_diff > 0) {
crm_info("Waiting up to %d seconds for cluster actions to complete", time_diff);
} else {
print_pending_actions(data_set.actions);
cleanup_alloc_calculations(&data_set);
return -ETIME;
}
if (rc == pcmk_ok) { /* this avoids sleep on first loop iteration */
sleep(WAIT_SLEEP_S);
}
/* Get latest transition graph */
cleanup_alloc_calculations(&data_set);
rc = update_working_set_from_cib(&data_set, cib);
if (rc != pcmk_ok) {
cleanup_alloc_calculations(&data_set);
return rc;
}
do_calculations(&data_set, data_set.input, NULL);
} while (actions_are_pending(data_set.actions));
return pcmk_ok;
}
int
cli_resource_execute(const char *rsc_id, const char *rsc_action, GHashTable *override_hash, cib_t * cib, pe_working_set_t *data_set)
{
int rc = pcmk_ok;
svc_action_t *op = NULL;
const char *rtype = NULL;
const char *rprov = NULL;
const char *rclass = NULL;
const char *action = NULL;
GHashTable *params = NULL;
resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
if (rsc == NULL) {
CMD_ERR("Must supply a resource id with -r");
return -ENXIO;
}
if (safe_str_eq(rsc_action, "force-check")) {
action = "monitor";
} else if (safe_str_eq(rsc_action, "force-stop")) {
action = rsc_action+6;
} else if (safe_str_eq(rsc_action, "force-start")
|| safe_str_eq(rsc_action, "force-demote")
|| safe_str_eq(rsc_action, "force-promote")) {
action = rsc_action+6;
if(rsc->variant >= pe_clone) {
rc = cli_resource_search(rsc_id, data_set);
if(rc > 0 && do_force == FALSE) {
CMD_ERR("It is not safe to %s %s here: the cluster claims it is already active", action, rsc_id);
CMD_ERR("Try setting target-role=stopped first or specifying --force");
crm_exit(EPERM);
}
}
}
if(rsc->variant == pe_clone || rsc->variant == pe_master) {
/* Grab the first child resource in the hope it's not a group */
rsc = rsc->children->data;
}
if(rsc->variant == pe_group) {
CMD_ERR("Sorry, --%s doesn't support group resources", rsc_action);
crm_exit(EOPNOTSUPP);
}
rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
if(safe_str_eq(rclass, "stonith")){
CMD_ERR("Sorry, --%s doesn't support %s resources yet", rsc_action, rclass);
crm_exit(EOPNOTSUPP);
}
params = generate_resource_params(rsc, data_set);
op = resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params, 0);
if(do_trace) {
setenv("OCF_TRACE_RA", "1", 1);
}
if(op && override_hash) {
GHashTableIter iter;
char *name = NULL;
char *value = NULL;
g_hash_table_iter_init(&iter, override_hash);
while (g_hash_table_iter_next(&iter, (gpointer *) & name, (gpointer *) & value)) {
printf("Overriding the cluster configuration for '%s' with '%s' = '%s'\n",
rsc->id, name, value);
g_hash_table_replace(op->params, strdup(name), strdup(value));
}
}
if(op == NULL) {
/* Re-run but with stderr enabled so we can display a sane error message */
crm_enable_stderr(TRUE);
resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params, 0);
return crm_exit(EINVAL);
} else if (services_action_sync(op)) {
int more, lpc, last;
char *local_copy = NULL;
if (op->status == PCMK_LRM_OP_DONE) {
printf("Operation %s for %s (%s:%s:%s) returned %d\n",
action, rsc->id, rclass, rprov ? rprov : "", rtype, op->rc);
} else {
printf("Operation %s for %s (%s:%s:%s) failed: %d\n",
action, rsc->id, rclass, rprov ? rprov : "", rtype, op->status);
}
if (op->stdout_data) {
local_copy = strdup(op->stdout_data);
more = strlen(local_copy);
last = 0;
for (lpc = 0; lpc < more; lpc++) {
if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
local_copy[lpc] = 0;
printf(" > stdout: %s\n", local_copy + last);
last = lpc + 1;
}
}
free(local_copy);
}
if (op->stderr_data) {
local_copy = strdup(op->stderr_data);
more = strlen(local_copy);
last = 0;
for (lpc = 0; lpc < more; lpc++) {
if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
local_copy[lpc] = 0;
printf(" > stderr: %s\n", local_copy + last);
last = lpc + 1;
}
}
free(local_copy);
}
}
rc = op->rc;
services_action_free(op);
return rc;
}
int
cli_resource_move(const char *rsc_id, const char *host_name, cib_t * cib, pe_working_set_t *data_set)
{
int rc = -EINVAL;
int count = 0;
node_t *current = NULL;
node_t *dest = pe_find_node(data_set->nodes, host_name);
resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
bool cur_is_dest = FALSE;
if (rsc == NULL) {
CMD_ERR("Resource '%s' not moved: not found", rsc_id);
return -ENXIO;
} else if (scope_master && rsc->variant < pe_master) {
resource_t *p = uber_parent(rsc);
if(p->variant == pe_master) {
CMD_ERR("Using parent '%s' for --move command instead of '%s'.", rsc->id, rsc_id);
rsc_id = p->id;
rsc = p;
} else {
CMD_ERR("Ignoring '--master' option: not valid for %s resources.",
get_resource_typename(rsc->variant));
scope_master = FALSE;
}
}
if(rsc->variant == pe_master) {
GListPtr iter = NULL;
for(iter = rsc->children; iter; iter = iter->next) {
resource_t *child = (resource_t *)iter->data;
enum rsc_role_e child_role = child->fns->state(child, TRUE);
if(child_role == RSC_ROLE_MASTER) {
rsc = child;
count++;
}
}
if(scope_master == FALSE && count == 0) {
count = g_list_length(rsc->running_on);
}
} else if (rsc->variant > pe_group) {
count = g_list_length(rsc->running_on);
} else if (g_list_length(rsc->running_on) > 1) {
CMD_ERR("Resource '%s' not moved: active on multiple nodes", rsc_id);
return rc;
}
if(dest == NULL) {
CMD_ERR("Error performing operation: node '%s' is unknown", host_name);
return -ENXIO;
}
if(g_list_length(rsc->running_on) == 1) {
current = rsc->running_on->data;
}
if(current == NULL) {
/* Nothing to check */
} else if(scope_master && rsc->fns->state(rsc, TRUE) != RSC_ROLE_MASTER) {
crm_trace("%s is already active on %s but not in correct state", rsc_id, dest->details->uname);
} else if (safe_str_eq(current->details->uname, dest->details->uname)) {
cur_is_dest = TRUE;
if (do_force) {
crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
rsc_id, scope_master?"promoted":"active", dest->details->uname);
} else {
CMD_ERR("Error performing operation: %s is already %s on %s",
rsc_id, scope_master?"promoted":"active", dest->details->uname);
return rc;
}
}
/* Clear any previous constraints for 'dest' */
cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib);
/* Record an explicit preference for 'dest' */
rc = cli_resource_prefer(rsc_id, dest->details->uname, cib);
crm_trace("%s%s now prefers node %s%s",
rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");
/* only ban the previous location if current location != destination location.
* it is possible to use -M to enforce a location without regard of where the
* resource is currently located */
if(do_force && (cur_is_dest == FALSE)) {
/* Ban the original location if possible */
if(current) {
(void)cli_resource_ban(rsc_id, current->details->uname, NULL, cib);
} else if(count > 1) {
CMD_ERR("Resource '%s' is currently %s in %d locations. One may now move one to %s",
rsc_id, scope_master?"promoted":"active", count, dest->details->uname);
CMD_ERR("You can prevent '%s' from being %s at a specific location with:"
" --ban %s--host <name>", rsc_id, scope_master?"promoted":"active", scope_master?"--master ":"");
} else {
- crm_trace("Not banning %s from it's current location: not active", rsc_id);
+ crm_trace("Not banning %s from its current location: not active", rsc_id);
}
}
return rc;
}