diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
index 9a1d094591..974eadfbf7 100644
--- a/daemons/controld/controld_fencing.c
+++ b/daemons/controld/controld_fencing.c
@@ -1,957 +1,959 @@
/*
* Copyright 2004-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <pacemaker-controld.h>
static void
tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event);
/*
* stonith failure counting
*
* We don't want to get stuck in a permanent fencing loop. Keep track of the
* number of fencing failures for each target node, and the most we'll restart a
* transition for.
*/
struct st_fail_rec {
int count;
};
static bool fence_reaction_panic = FALSE;
static unsigned long int stonith_max_attempts = 10;
static GHashTable *stonith_failures = NULL;
// crmd_opts defines default for stonith-max-attempts, so value is never NULL
void
update_stonith_max_attempts(const char *value)
{
if (safe_str_eq(value, CRM_INFINITY_S)) {
stonith_max_attempts = CRM_SCORE_INFINITY;
} else {
stonith_max_attempts = (unsigned long int) crm_parse_ll(value, NULL);
}
}
void
set_fence_reaction(const char *reaction_s)
{
if (safe_str_eq(reaction_s, "panic")) {
fence_reaction_panic = TRUE;
} else {
if (safe_str_neq(reaction_s, "stop")) {
crm_warn("Invalid value '%s' for %s, using 'stop'",
reaction_s, XML_CONFIG_ATTR_FENCE_REACTION);
}
fence_reaction_panic = FALSE;
}
}
static gboolean
too_many_st_failures(const char *target)
{
GHashTableIter iter;
const char *key = NULL;
struct st_fail_rec *value = NULL;
if (stonith_failures == NULL) {
return FALSE;
}
if (target == NULL) {
g_hash_table_iter_init(&iter, stonith_failures);
while (g_hash_table_iter_next(&iter, (gpointer *) &key,
(gpointer *) &value)) {
if (value->count >= stonith_max_attempts) {
target = (const char*)key;
goto too_many;
}
}
} else {
value = g_hash_table_lookup(stonith_failures, target);
if ((value != NULL) && (value->count >= stonith_max_attempts)) {
goto too_many;
}
}
return FALSE;
too_many:
crm_warn("Too many failures (%d) to fence %s, giving up",
value->count, target);
return TRUE;
}
/*!
* \internal
* \brief Reset a stonith fail count
*
* \param[in] target Name of node to reset, or NULL for all
*/
void
st_fail_count_reset(const char *target)
{
if (stonith_failures == NULL) {
return;
}
if (target) {
struct st_fail_rec *rec = NULL;
rec = g_hash_table_lookup(stonith_failures, target);
if (rec) {
rec->count = 0;
}
} else {
GHashTableIter iter;
const char *key = NULL;
struct st_fail_rec *rec = NULL;
g_hash_table_iter_init(&iter, stonith_failures);
while (g_hash_table_iter_next(&iter, (gpointer *) &key,
(gpointer *) &rec)) {
rec->count = 0;
}
}
}
static void
st_fail_count_increment(const char *target)
{
struct st_fail_rec *rec = NULL;
if (stonith_failures == NULL) {
stonith_failures = crm_str_table_new();
}
rec = g_hash_table_lookup(stonith_failures, target);
if (rec) {
rec->count++;
} else {
rec = malloc(sizeof(struct st_fail_rec));
if(rec == NULL) {
return;
}
rec->count = 1;
g_hash_table_insert(stonith_failures, strdup(target), rec);
}
}
/* end stonith fail count functions */
static void
cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output,
void *user_data)
{
if (rc < pcmk_ok) {
crm_err("Fencing update %d for %s: failed - %s (%d)",
call_id, (char *)user_data, pcmk_strerror(rc), rc);
crm_log_xml_warn(msg, "Failed update");
abort_transition(INFINITY, tg_shutdown, "CIB update failed", NULL);
} else {
crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
}
}
static void
send_stonith_update(crm_action_t *action, const char *target, const char *uuid)
{
int rc = pcmk_ok;
crm_node_t *peer = NULL;
/* We (usually) rely on the membership layer to do node_update_cluster,
* and the peer status callback to do node_update_peer, because the node
* might have already rejoined before we get the stonith result here.
*/
int flags = node_update_join | node_update_expected;
/* zero out the node-status & remove all LRM status info */
xmlNode *node_state = NULL;
CRM_CHECK(target != NULL, return);
CRM_CHECK(uuid != NULL, return);
/* Make sure the membership and join caches are accurate */
peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);
CRM_CHECK(peer != NULL, return);
if (peer->state == NULL) {
/* Usually, we rely on the membership layer to update the cluster state
* in the CIB. However, if the node has never been seen, do it here, so
* the node is not considered unclean.
*/
flags |= node_update_cluster;
}
if (peer->uuid == NULL) {
crm_info("Recording uuid '%s' for node '%s'", uuid, target);
peer->uuid = strdup(uuid);
}
crmd_peer_down(peer, TRUE);
/* Generate a node state update for the CIB */
node_state = create_node_state_update(peer, flags, NULL, __FUNCTION__);
/* we have to mark whether or not remote nodes have already been fenced */
if (peer->flags & crm_remote_node) {
time_t now = time(NULL);
char *now_s = crm_itoa(now);
crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
free(now_s);
}
/* Force our known ID */
crm_xml_add(node_state, XML_ATTR_UUID, uuid);
rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
cib_quorum_override | cib_scope_local | cib_can_create);
/* Delay processing the trigger until the update completes */
crm_debug("Sending fencing update %d for %s", rc, target);
fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated);
/* Make sure it sticks */
/* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local); */
controld_delete_node_state(peer->uname, controld_section_all,
cib_scope_local);
free_xml(node_state);
return;
}
/*!
* \internal
* \brief Abort transition due to stonith failure
*
* \param[in] abort_action Whether to restart or stop transition
* \param[in] target Don't restart if this (NULL for any) has too many failures
* \param[in] reason Log this stonith action XML as abort reason (or NULL)
*/
static void
abort_for_stonith_failure(enum transition_action abort_action,
const char *target, xmlNode *reason)
{
/* If stonith repeatedly fails, we eventually give up on starting a new
* transition for that reason.
*/
if ((abort_action != tg_stop) && too_many_st_failures(target)) {
abort_action = tg_stop;
}
abort_transition(INFINITY, abort_action, "Stonith failed", reason);
}
/*
* stonith cleanup list
*
* If the DC is shot, proper notifications might not go out.
* The stonith cleanup list allows the cluster to (re-)send
* notifications once a new DC is elected.
*/
static GListPtr stonith_cleanup_list = NULL;
/*!
* \internal
* \brief Add a node to the stonith cleanup list
*
* \param[in] target Name of node to add
*/
void
add_stonith_cleanup(const char *target) {
stonith_cleanup_list = g_list_append(stonith_cleanup_list, strdup(target));
}
/*!
* \internal
* \brief Remove a node from the stonith cleanup list
*
* \param[in] Name of node to remove
*/
void
remove_stonith_cleanup(const char *target)
{
GListPtr iter = stonith_cleanup_list;
while (iter != NULL) {
GListPtr tmp = iter;
char *iter_name = tmp->data;
iter = iter->next;
if (safe_str_eq(target, iter_name)) {
crm_trace("Removing %s from the cleanup list", iter_name);
stonith_cleanup_list = g_list_delete_link(stonith_cleanup_list, tmp);
free(iter_name);
}
}
}
/*!
* \internal
* \brief Purge all entries from the stonith cleanup list
*/
void
purge_stonith_cleanup()
{
if (stonith_cleanup_list) {
GListPtr iter = NULL;
for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
char *target = iter->data;
crm_info("Purging %s from stonith cleanup list", target);
free(target);
}
g_list_free(stonith_cleanup_list);
stonith_cleanup_list = NULL;
}
}
/*!
* \internal
* \brief Send stonith updates for all entries in cleanup list, then purge it
*/
void
execute_stonith_cleanup()
{
GListPtr iter;
for (iter = stonith_cleanup_list; iter != NULL; iter = iter->next) {
char *target = iter->data;
crm_node_t *target_node = crm_get_peer(0, target);
const char *uuid = crm_peer_uuid(target_node);
crm_notice("Marking %s, target of a previous stonith action, as clean", target);
send_stonith_update(NULL, target, uuid);
free(target);
}
g_list_free(stonith_cleanup_list);
stonith_cleanup_list = NULL;
}
/* end stonith cleanup list functions */
/* stonith API client
*
* Functions that need to interact directly with the fencer via its API
*/
static stonith_t *stonith_api = NULL;
static crm_trigger_t *stonith_reconnect = NULL;
static char *te_client_id = NULL;
static gboolean
fail_incompletable_stonith(crm_graph_t *graph)
{
GListPtr lpc = NULL;
const char *task = NULL;
xmlNode *last_action = NULL;
if (graph == NULL) {
return FALSE;
}
for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
GListPtr lpc2 = NULL;
synapse_t *synapse = (synapse_t *) lpc->data;
if (synapse->confirmed) {
continue;
}
for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
crm_action_t *action = (crm_action_t *) lpc2->data;
if (action->type != action_type_crm || action->confirmed) {
continue;
}
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
if (task && safe_str_eq(task, CRM_OP_FENCE)) {
action->failed = TRUE;
last_action = action->xml;
update_graph(graph, action);
crm_notice("Failing action %d (%s): fencer terminated",
action->id, ID(action->xml));
}
}
}
if (last_action != NULL) {
crm_warn("Fencer failure resulted in unrunnable actions");
abort_for_stonith_failure(tg_restart, NULL, last_action);
return TRUE;
}
return FALSE;
}
static void
tengine_stonith_connection_destroy(stonith_t *st, stonith_event_t *e)
{
te_cleanup_stonith_history_sync(st, FALSE);
if (is_set(fsa_input_register, R_ST_REQUIRED)) {
crm_crit("Fencing daemon connection failed");
mainloop_set_trigger(stonith_reconnect);
} else {
crm_info("Fencing daemon disconnected");
}
if (stonith_api) {
/* the client API won't properly reconnect notifications
* if they are still in the table - so remove them
*/
if (stonith_api->state != stonith_disconnected) {
stonith_api->cmds->disconnect(st);
}
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE);
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED);
}
if (AM_I_DC) {
fail_incompletable_stonith(transition_graph);
trigger_graph();
}
}
static void
tengine_stonith_notify(stonith_t *st, stonith_event_t *st_event)
{
if (te_client_id == NULL) {
te_client_id = crm_strdup_printf("%s.%lu", crm_system_name,
(unsigned long) getpid());
}
if (st_event == NULL) {
crm_err("Notify data not found");
return;
}
crmd_alert_fencing_op(st_event);
if ((st_event->result == pcmk_ok) && safe_str_eq("on", st_event->action)) {
crm_notice("%s was successfully unfenced by %s (at the request of %s)",
st_event->target,
st_event->executioner? st_event->executioner : "<anyone>",
st_event->origin);
/* TODO: Hook up st_event->device */
return;
} else if (safe_str_eq("on", st_event->action)) {
crm_err("Unfencing of %s by %s failed: %s (%d)",
st_event->target,
st_event->executioner? st_event->executioner : "<anyone>",
pcmk_strerror(st_event->result), st_event->result);
return;
} else if ((st_event->result == pcmk_ok)
&& crm_str_eq(st_event->target, fsa_our_uname, TRUE)) {
/* We were notified of our own fencing. Most likely, either fencing was
* misconfigured, or fabric fencing that doesn't cut cluster
* communication is in use.
*
* Either way, shutting down the local host is a good idea, to require
* administrator intervention. Also, other nodes would otherwise likely
* set our status to lost because of the fencing callback and discard
* our subsequent election votes as "not part of our cluster".
*/
crm_crit("We were allegedly just fenced by %s for %s!",
st_event->executioner? st_event->executioner : "the cluster",
st_event->origin); /* Dumps blackbox if enabled */
if (fence_reaction_panic) {
pcmk_panic(__FUNCTION__);
} else {
crm_exit(CRM_EX_FATAL);
}
return;
}
/* Update the count of stonith failures for this target, in case we become
* DC later. The current DC has already updated its fail count in
* tengine_stonith_callback().
*/
if (!AM_I_DC && safe_str_eq(st_event->operation, T_STONITH_NOTIFY_FENCE)) {
if (st_event->result == pcmk_ok) {
st_fail_count_reset(st_event->target);
} else {
st_fail_count_increment(st_event->target);
}
}
crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s: %s "
CRM_XS " initiator=%s ref=%s",
st_event->target, st_event->result == pcmk_ok ? "" : " not",
st_event->action,
st_event->executioner ? st_event->executioner : "<anyone>",
(st_event->client_origin? st_event->client_origin : "<unknown>"),
pcmk_strerror(st_event->result),
st_event->origin, st_event->id);
if (st_event->result == pcmk_ok) {
crm_node_t *peer = crm_find_known_peer_full(0, st_event->target, CRM_GET_PEER_ANY);
const char *uuid = NULL;
gboolean we_are_executioner = safe_str_eq(st_event->executioner, fsa_our_uname);
if (peer == NULL) {
return;
}
uuid = crm_peer_uuid(peer);
crm_trace("target=%s dc=%s", st_event->target, fsa_our_dc);
if(AM_I_DC) {
/* The DC always sends updates */
send_stonith_update(NULL, st_event->target, uuid);
/* @TODO Ideally, at this point, we'd check whether the fenced node
* hosted any guest nodes, and call remote_node_down() for them.
* Unfortunately, the controller doesn't have a simple, reliable way
* to map hosts to guests. It might be possible to track this in the
* peer cache via crm_remote_peer_cache_refresh(). For now, we rely
* on the scheduler creating fence pseudo-events for the guests.
*/
if (st_event->client_origin
&& safe_str_neq(st_event->client_origin, te_client_id)) {
/* Abort the current transition graph if it wasn't us
* that invoked stonith to fence someone
*/
crm_info("External fencing operation from %s fenced %s", st_event->client_origin, st_event->target);
abort_transition(INFINITY, tg_restart, "External Fencing Operation", NULL);
}
/* Assume it was our leader if we don't currently have one */
} else if (((fsa_our_dc == NULL) || safe_str_eq(fsa_our_dc, st_event->target))
&& is_not_set(peer->flags, crm_remote_node)) {
crm_notice("Fencing target %s %s our leader",
st_event->target, (fsa_our_dc? "was" : "may have been"));
/* Given the CIB resyncing that occurs around elections,
* have one node update the CIB now and, if the new DC is different,
* have them do so too after the election
*/
if (we_are_executioner) {
send_stonith_update(NULL, st_event->target, uuid);
}
add_stonith_cleanup(st_event->target);
}
/* If the target is a remote node, and we host its connection,
* immediately fail all monitors so it can be recovered quickly.
* The connection won't necessarily drop when a remote node is fenced,
* so the failure might not otherwise be detected until the next poke.
*/
if (is_set(peer->flags, crm_remote_node)) {
remote_ra_fail(st_event->target);
}
crmd_peer_down(peer, TRUE);
}
}
/*!
* \brief Connect to fencer
*
* \param[in] user_data If NULL, retry failures now, otherwise retry in main loop
*
* \return TRUE
* \note If user_data is NULL, this will wait 2s between attempts, for up to
* 30 attempts, meaning the controller could be blocked as long as 58s.
*/
static gboolean
te_connect_stonith(gpointer user_data)
{
int rc = pcmk_ok;
if (stonith_api == NULL) {
stonith_api = stonith_api_new();
if (stonith_api == NULL) {
crm_err("Could not connect to fencer: API memory allocation failed");
return TRUE;
}
}
if (stonith_api->state != stonith_disconnected) {
crm_trace("Already connected to fencer, no need to retry");
return TRUE;
}
if (user_data == NULL) {
// Blocking (retry failures now until successful)
rc = stonith_api_connect_retry(stonith_api, crm_system_name, 30);
if (rc != pcmk_ok) {
crm_err("Could not connect to fencer in 30 attempts: %s "
CRM_XS " rc=%d", pcmk_strerror(rc), rc);
}
} else {
// Non-blocking (retry failures later in main loop)
rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL);
if (rc != pcmk_ok) {
if (is_set(fsa_input_register, R_ST_REQUIRED)) {
crm_notice("Fencer connection failed (will retry): %s "
CRM_XS " rc=%d", pcmk_strerror(rc), rc);
mainloop_set_trigger(stonith_reconnect);
} else {
crm_info("Fencer connection failed (ignoring because no longer required): %s "
CRM_XS " rc=%d", pcmk_strerror(rc), rc);
}
return TRUE;
}
}
if (rc == pcmk_ok) {
stonith_api->cmds->register_notification(stonith_api,
T_STONITH_NOTIFY_DISCONNECT,
tengine_stonith_connection_destroy);
stonith_api->cmds->register_notification(stonith_api,
T_STONITH_NOTIFY_FENCE,
tengine_stonith_notify);
stonith_api->cmds->register_notification(stonith_api,
T_STONITH_NOTIFY_HISTORY_SYNCED,
tengine_stonith_history_synced);
te_trigger_stonith_history_sync(TRUE);
crm_notice("Fencer successfully connected");
}
return TRUE;
}
/*!
\internal
\brief Schedule fencer connection attempt in main loop
*/
void
controld_trigger_fencer_connect()
{
if (stonith_reconnect == NULL) {
stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW,
te_connect_stonith,
GINT_TO_POINTER(TRUE));
}
set_bit(fsa_input_register, R_ST_REQUIRED);
mainloop_set_trigger(stonith_reconnect);
}
void
controld_disconnect_fencer(bool destroy)
{
if (stonith_api) {
// Prevent fencer connection from coming up again
clear_bit(fsa_input_register, R_ST_REQUIRED);
if (stonith_api->state != stonith_disconnected) {
stonith_api->cmds->disconnect(stonith_api);
}
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE);
stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED);
}
if (destroy) {
if (stonith_api) {
stonith_api->cmds->free(stonith_api);
stonith_api = NULL;
}
if (stonith_reconnect) {
mainloop_destroy_trigger(stonith_reconnect);
stonith_reconnect = NULL;
}
if (te_client_id) {
free(te_client_id);
te_client_id = NULL;
}
}
}
static gboolean
do_stonith_history_sync(gpointer user_data)
{
if (stonith_api && (stonith_api->state != stonith_disconnected)) {
stonith_history_t *history = NULL;
te_cleanup_stonith_history_sync(stonith_api, FALSE);
stonith_api->cmds->history(stonith_api,
st_opt_sync_call | st_opt_broadcast,
NULL, &history, 5);
stonith_history_free(history);
return TRUE;
} else {
crm_info("Skip triggering stonith history-sync as stonith is disconnected");
return FALSE;
}
}
static void
tengine_stonith_callback(stonith_t *stonith, stonith_callback_data_t *data)
{
char *uuid = NULL;
int stonith_id = -1;
int transition_id = -1;
crm_action_t *action = NULL;
int call_id = data->call_id;
int rc = data->rc;
char *userdata = data->userdata;
CRM_CHECK(userdata != NULL, return);
crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata,
pcmk_strerror(rc), rc);
if (AM_I_DC == FALSE) {
return;
}
/* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */
/* op->call_id, op->optype, op->node_name, op->op_result, */
/* (char *)op->node_list, op->private_data); */
/* filter out old STONITH actions */
CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, NULL),
goto bail);
if (transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid)
|| transition_graph->id != transition_id) {
crm_info("Ignoring STONITH action initiated outside of the current transition");
goto bail;
}
action = controld_get_action(stonith_id);
if (action == NULL) {
crm_err("Stonith action not matched");
goto bail;
}
stop_te_timer(action->timer);
if (rc == pcmk_ok) {
const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
const char *op = crm_meta_value(action->params, "stonith_action");
crm_info("Stonith operation %d for %s passed", call_id, target);
if (action->confirmed == FALSE) {
te_action_confirmed(action, NULL);
if (safe_str_eq("on", op)) {
const char *value = NULL;
char *now = crm_ttoa(time(NULL));
update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE);
free(now);
value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE);
value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE);
} else if (action->sent_update == FALSE) {
send_stonith_update(action, target, uuid);
action->sent_update = TRUE;
}
}
st_fail_count_reset(target);
} else {
const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
enum transition_action abort_action = tg_restart;
action->failed = TRUE;
crm_notice("Stonith operation %d for %s failed (%s): aborting transition.",
call_id, target, pcmk_strerror(rc));
/* If no fence devices were available, there's no use in immediately
* checking again, so don't start a new transition in that case.
*/
if (rc == -ENODEV) {
crm_warn("No devices found in cluster to fence %s, giving up",
target);
abort_action = tg_stop;
}
/* Increment the fail count now, so abort_for_stonith_failure() can
* check it. Non-DC nodes will increment it in tengine_stonith_notify().
*/
st_fail_count_increment(target);
abort_for_stonith_failure(abort_action, target, NULL);
}
update_graph(transition_graph, action);
trigger_graph();
bail:
free(userdata);
free(uuid);
return;
}
gboolean
te_fence_node(crm_graph_t *graph, crm_action_t *action)
{
int rc = 0;
const char *id = NULL;
const char *uuid = NULL;
const char *target = NULL;
const char *type = NULL;
char *transition_key = NULL;
const char *priority_delay = NULL;
gboolean invalid_action = FALSE;
enum stonith_call_options options = st_opt_none;
id = ID(action->xml);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
type = crm_meta_value(action->params, "stonith_action");
CRM_CHECK(id != NULL, invalid_action = TRUE);
CRM_CHECK(uuid != NULL, invalid_action = TRUE);
CRM_CHECK(type != NULL, invalid_action = TRUE);
CRM_CHECK(target != NULL, invalid_action = TRUE);
if (invalid_action) {
crm_log_xml_warn(action->xml, "BadAction");
return FALSE;
}
priority_delay = crm_meta_value(action->params, XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
crm_notice("Requesting fencing (%s) of node %s "
- CRM_XS " action=%s timeout=%u priority_delay=%s",
- type, target, id, transition_graph->stonith_timeout, priority_delay);
+ CRM_XS " action=%s timeout=%u%s%s",
+ type, target, id, transition_graph->stonith_timeout,
+ priority_delay ? " priority_delay=" : "",
+ priority_delay ? priority_delay : "");
/* Passing NULL means block until we can connect... */
te_connect_stonith(NULL);
if (crmd_join_phase_count(crm_join_confirmed) == 1) {
options |= st_opt_allow_suicide;
}
rc = stonith_api->cmds->fence_with_delay(stonith_api, options, target, type,
(int) (transition_graph->stonith_timeout / 1000),
0, crm_atoi(priority_delay, "0"));
transition_key = pcmk__transition_key(transition_graph->id, action->id, 0,
te_uuid),
stonith_api->cmds->register_callback(stonith_api, rc,
(int) (transition_graph->stonith_timeout / 1000),
st_opt_timeout_updates, transition_key,
"tengine_stonith_callback", tengine_stonith_callback);
return TRUE;
}
/* end stonith API client functions */
/*
* stonith history synchronization
*
* Each node's fencer keeps track of a cluster-wide fencing history. When a node
* joins or leaves, we need to synchronize the history across all nodes.
*/
static crm_trigger_t *stonith_history_sync_trigger = NULL;
static mainloop_timer_t *stonith_history_sync_timer_short = NULL;
static mainloop_timer_t *stonith_history_sync_timer_long = NULL;
void
te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers)
{
if (free_timers) {
mainloop_timer_del(stonith_history_sync_timer_short);
stonith_history_sync_timer_short = NULL;
mainloop_timer_del(stonith_history_sync_timer_long);
stonith_history_sync_timer_long = NULL;
} else {
mainloop_timer_stop(stonith_history_sync_timer_short);
mainloop_timer_stop(stonith_history_sync_timer_long);
}
if (st) {
st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED);
}
}
static void
tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event)
{
te_cleanup_stonith_history_sync(st, FALSE);
crm_debug("Fence-history synced - cancel all timers");
}
static gboolean
stonith_history_sync_set_trigger(gpointer user_data)
{
mainloop_set_trigger(stonith_history_sync_trigger);
return FALSE;
}
void
te_trigger_stonith_history_sync(bool long_timeout)
{
/* trigger a sync in 5s to give more nodes the
* chance to show up so that we don't create
* unnecessary stonith-history-sync traffic
*
* the long timeout of 30s is there as a fallback
* so that after a successful connection to fenced
* we will wait for 30s for the DC to trigger a
* history-sync
* if this doesn't happen we trigger a sync locally
* (e.g. fenced segfaults and is restarted by pacemakerd)
*/
/* as we are finally checking the stonith-connection
* in do_stonith_history_sync we should be fine
* leaving stonith_history_sync_time & stonith_history_sync_trigger
* around
*/
if (stonith_history_sync_trigger == NULL) {
stonith_history_sync_trigger =
mainloop_add_trigger(G_PRIORITY_LOW,
do_stonith_history_sync, NULL);
}
if (long_timeout) {
if(stonith_history_sync_timer_long == NULL) {
stonith_history_sync_timer_long =
mainloop_timer_add("history_sync_long", 30000,
FALSE, stonith_history_sync_set_trigger,
NULL);
}
crm_info("Fence history will be synchronized cluster-wide within 30 seconds");
mainloop_timer_start(stonith_history_sync_timer_long);
} else {
if(stonith_history_sync_timer_short == NULL) {
stonith_history_sync_timer_short =
mainloop_timer_add("history_sync_short", 5000,
FALSE, stonith_history_sync_set_trigger,
NULL);
}
crm_info("Fence history will be synchronized cluster-wide within 5 seconds");
mainloop_timer_start(stonith_history_sync_timer_short);
}
}
/* end stonith history synchronization functions */
diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c
index 1a2adb4bb7..1273a57819 100644
--- a/tools/stonith_admin.c
+++ b/tools/stonith_admin.c
@@ -1,631 +1,631 @@
/*
* Copyright 2009-2020 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/utsname.h>
#include <errno.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/ipc.h>
#include <crm/cluster/internal.h>
#include <crm/common/cmdline_internal.h>
#include <crm/common/output.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h>
#include <crm/cib.h>
#include <crm/pengine/status.h>
#include <crm/common/xml.h>
#include <pacemaker-internal.h>
#define SUMMARY "stonith_admin - Access the Pacemaker fencing API"
char action = 0;
struct {
gboolean as_nodeid;
gboolean broadcast;
gboolean cleanup;
gboolean installed;
gboolean metadata;
gboolean registered;
gboolean validate_cfg;
stonith_key_value_t *devices;
stonith_key_value_t *params;
int fence_level;
int timeout ;
int tolerance;
int delay;
char *agent;
char *confirm_host;
char *fence_host;
char *history;
char *last_fenced;
char *query;
char *reboot_host;
char *register_dev;
char *register_level;
char *targets;
char *terminate;
char *unfence_host;
char *unregister_dev;
char *unregister_level;
} options = {
.timeout = 120,
.delay = 0
};
gboolean add_env_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean add_stonith_device(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean add_stonith_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean add_tolerance(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean set_tag(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
#define INDENT " "
/* *INDENT-OFF* */
static GOptionEntry defn_entries[] = {
{ "register", 'R', 0, G_OPTION_ARG_STRING, &options.register_dev,
"Register the named stonith device. Requires: --agent.\n"
INDENT "Optional: --option, --env-option.",
"DEVICE" },
{ "deregister", 'D', 0, G_OPTION_ARG_STRING, &options.unregister_dev,
"De-register the named stonith device.",
"DEVICE" },
{ "register-level", 'r', 0, G_OPTION_ARG_STRING, &options.register_level,
"Register a stonith level for the named target,\n"
INDENT "specified as one of NAME, @PATTERN, or ATTR=VALUE.\n"
INDENT "Requires: --index and one or more --device entries.",
"TARGET" },
{ "deregister-level", 'd', 0, G_OPTION_ARG_STRING, &options.unregister_level,
"Unregister a stonith level for the named target,\n"
INDENT "specified as for --register-level. Requires: --index",
"TARGET" },
{ NULL }
};
static GOptionEntry query_entries[] = {
{ "list", 'l', 0, G_OPTION_ARG_STRING, &options.terminate,
"List devices that can terminate the specified host.\n"
INDENT "Optional: --timeout",
"HOST" },
{ "list-registered", 'L', 0, G_OPTION_ARG_NONE, &options.registered,
"List all registered devices. Optional: --timeout.",
NULL },
{ "list-installed", 'I', 0, G_OPTION_ARG_NONE, &options.installed,
"List all installed devices. Optional: --timeout.",
NULL },
{ "list-targets", 's', 0, G_OPTION_ARG_STRING, &options.targets,
"List the targets that can be fenced by the\n"
INDENT "named device. Optional: --timeout.",
"DEVICE" },
{ "metadata", 'M', 0, G_OPTION_ARG_NONE, &options.metadata,
"Show agent metadata. Requires: --agent.\n"
INDENT "Optional: --timeout.",
NULL },
{ "query", 'Q', 0, G_OPTION_ARG_STRING, &options.query,
"Check the named device's status. Optional: --timeout.",
"DEVICE" },
{ "history", 'H', 0, G_OPTION_ARG_STRING, &options.history,
"Show last successful fencing operation for named node\n"
INDENT "(or '*' for all nodes). Optional: --timeout, --cleanup,\n"
INDENT "--quiet (show only the operation's epoch timestamp),\n"
INDENT "--verbose (show all recorded and pending operations),\n"
INDENT "--broadcast (update history from all nodes available).",
"NODE" },
{ "last", 'h', 0, G_OPTION_ARG_STRING, &options.last_fenced,
"Indicate when the named node was last fenced.\n"
INDENT "Optional: --as-node-id.",
"NODE" },
{ "validate", 'K', 0, G_OPTION_ARG_NONE, &options.validate_cfg,
"Validate a fence device configuration.\n"
INDENT "Requires: --agent. Optional: --option, --env-option,\n"
INDENT "--quiet (print no output, only return status).",
NULL },
{ NULL }
};
static GOptionEntry fence_entries[] = {
{ "fence", 'F', 0, G_OPTION_ARG_STRING, &options.fence_host,
- "Fence named host. Optional: --timeout, --tolerance.",
+ "Fence named host. Optional: --timeout, --tolerance, --delay.",
"HOST" },
{ "unfence", 'U', 0, G_OPTION_ARG_STRING, &options.unfence_host,
- "Unfence named host. Optional: --timeout, --tolerance.",
+ "Unfence named host. Optional: --timeout, --tolerance, --delay.",
"HOST" },
{ "reboot", 'B', 0, G_OPTION_ARG_STRING, &options.reboot_host,
- "Reboot named host. Optional: --timeout, --tolerance.",
+ "Reboot named host. Optional: --timeout, --tolerance, --delay.",
"HOST" },
{ "confirm", 'C', 0, G_OPTION_ARG_STRING, &options.confirm_host,
"Tell cluster that named host is now safely down.",
"HOST", },
{ NULL }
};
static GOptionEntry addl_entries[] = {
{ "cleanup", 'c', 0, G_OPTION_ARG_NONE, &options.cleanup,
"Cleanup wherever appropriate. Requires --history.",
NULL },
{ "broadcast", 'b', 0, G_OPTION_ARG_NONE, &options.broadcast,
"Broadcast wherever appropriate.",
NULL },
{ "agent", 'a', 0, G_OPTION_ARG_STRING, &options.agent,
"The agent to use (for example, fence_xvm;\n"
INDENT "with --register, --metadata, --validate).",
"AGENT" },
{ "option", 'o', 0, G_OPTION_ARG_CALLBACK, add_stonith_params,
"Specify a device configuration parameter as NAME=VALUE\n"
INDENT "(may be specified multiple times; with --register,\n"
INDENT "--validate).",
"PARAM" },
{ "env-option", 'e', 0, G_OPTION_ARG_CALLBACK, add_env_params,
"Specify a device configuration parameter with the\n"
INDENT "specified name, using the value of the\n"
INDENT "environment variable of the same name prefixed with\n"
INDENT "OCF_RESKEY_ (may be specified multiple times;\n"
INDENT "with --register, --validate).",
"PARAM" },
{ "tag", 'T', 0, G_OPTION_ARG_CALLBACK, set_tag,
"Identify fencing operations in logs with the specified\n"
INDENT "tag; useful when multiple entities might invoke\n"
INDENT "stonith_admin (used with most commands).",
"TAG" },
{ "device", 'v', 0, G_OPTION_ARG_CALLBACK, add_stonith_device,
"Device ID (with --register-level, device to associate with\n"
INDENT "a given host and level; may be specified multiple times)"
#if SUPPORT_CIBSECRETS
"\n" INDENT "(with --validate, name to use to load CIB secrets)"
#endif
".",
"DEVICE" },
{ "index", 'i', 0, G_OPTION_ARG_INT, &options.fence_level,
"The stonith level (1-9) (with --register-level,\n"
INDENT "--deregister-level).",
"LEVEL" },
{ "timeout", 't', 0, G_OPTION_ARG_INT, &options.timeout,
"Operation timeout in seconds (default 120;\n"
INDENT "used with most commands).",
"SECONDS" },
{ "delay", 'y', 0, G_OPTION_ARG_INT, &options.delay,
"Apply a fencing delay in seconds. Any static/random delays from\n"
INDENT "pcmk_delay_base/max will be added, otherwise all\n"
INDENT "disabled with the value -1\n"
INDENT "(default 0; with --fence, --reboot, --unfence).",
"SECONDS" },
{ "as-node-id", 'n', 0, G_OPTION_ARG_NONE, &options.as_nodeid,
"(Advanced) The supplied node is the corosync node ID\n"
INDENT "(with --last).",
NULL },
{ "tolerance", 0, 0, G_OPTION_ARG_CALLBACK, add_tolerance,
"(Advanced) Do nothing if an equivalent --fence request\n"
INDENT "succeeded less than this many seconds earlier\n"
INDENT "(with --fence, --unfence, --reboot).",
"SECONDS" },
{ NULL }
};
/* *INDENT-ON* */
static pcmk__supported_format_t formats[] = {
PCMK__SUPPORTED_FORMAT_HTML,
PCMK__SUPPORTED_FORMAT_TEXT,
PCMK__SUPPORTED_FORMAT_XML,
{ NULL, NULL, NULL }
};
static int st_opts = st_opt_sync_call | st_opt_allow_suicide;
static char *name = NULL;
gboolean
add_env_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
char *key = crm_strdup_printf("OCF_RESKEY_%s", optarg);
const char *env = getenv(key);
gboolean retval = TRUE;
if (env == NULL) {
crm_err("Invalid option: -e %s", optarg);
g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM, "Invalid option: -e %s", optarg);
retval = FALSE;
} else {
crm_info("Got: '%s'='%s'", optarg, env);
options.params = stonith_key_value_add(options.params, optarg, env);
}
free(key);
return retval;
}
gboolean
add_stonith_device(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
options.devices = stonith_key_value_add(options.devices, NULL, optarg);
return TRUE;
}
gboolean
add_tolerance(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
options.tolerance = crm_get_msec(optarg) / 1000;
return TRUE;
}
gboolean
add_stonith_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
char *name = NULL;
char *value = NULL;
int rc = 0;
gboolean retval = TRUE;
crm_info("Scanning: -o %s", optarg);
rc = pcmk_scan_nvpair(optarg, &name, &value);
if (rc != 2) {
crm_err("Invalid option: -o %s: %s", optarg, pcmk_strerror(rc));
g_set_error(error, G_OPTION_ERROR, rc, "Invalid option: -o %s: %s", optarg, pcmk_strerror(rc));
retval = FALSE;
} else {
crm_info("Got: '%s'='%s'", name, value);
options.params = stonith_key_value_add(options.params, name, value);
}
free(name);
free(value);
return retval;
}
gboolean
set_tag(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
free(name);
name = crm_strdup_printf("%s.%s", crm_system_name, optarg);
return TRUE;
}
static GOptionContext *
build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
GOptionContext *context = NULL;
GOptionEntry extra_prog_entries[] = {
{ "quiet", 'q', 0, G_OPTION_ARG_NONE, &(args->quiet),
"Be less descriptive in output.",
NULL },
{ NULL }
};
context = pcmk__build_arg_context(args, "text (default), html, xml", group, NULL);
/* Add the -q option, which cannot be part of the globally supported options
* because some tools use that flag for something else.
*/
pcmk__add_main_args(context, extra_prog_entries);
pcmk__add_arg_group(context, "definition", "Device Definition Commands:",
"Show device definition help", defn_entries);
pcmk__add_arg_group(context, "queries", "Queries:",
"Show query help", query_entries);
pcmk__add_arg_group(context, "fence", "Fencing Commands:",
"Show fence help", fence_entries);
pcmk__add_arg_group(context, "additional", "Additional Options:",
"Show additional options", addl_entries);
return context;
}
int
main(int argc, char **argv)
{
int rc = 0;
bool no_connect = false;
bool required_agent = false;
char *target = NULL;
const char *device = NULL;
crm_exit_t exit_code = CRM_EX_OK;
stonith_t *st = NULL;
pcmk__output_t *out = NULL;
pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
GError *error = NULL;
GOptionContext *context = NULL;
GOptionGroup *output_group = NULL;
gchar **processed_args = NULL;
context = build_arg_context(args, &output_group);
pcmk__register_formats(output_group, formats);
crm_log_cli_init("stonith_admin");
name = strdup(crm_system_name);
processed_args = pcmk__cmdline_preproc(argv, "adehilorstvBCDFHQRTU");
if (!g_option_context_parse_strv(context, &processed_args, &error)) {
fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
exit_code = CRM_EX_USAGE;
goto done;
}
for (int i = 0; i < args->verbosity; i++) {
crm_bump_log_level(argc, argv);
}
rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
if (rc != pcmk_rc_ok) {
fprintf(stderr, "Error creating output format %s: %s\n",
args->output_ty, pcmk_rc_str(rc));
exit_code = CRM_EX_ERROR;
goto done;
}
stonith__register_messages(out);
if (args->version) {
out->version(out, false);
goto done;
}
if (options.validate_cfg) {
required_agent = true;
no_connect = true;
action = 'K';
}
if (options.installed) {
no_connect = true;
action = 'I';
}
if (options.registered) {
action = 'L';
}
if (options.register_dev != NULL) {
required_agent = true;
action = 'R';
device = options.register_dev;
}
if (options.query != NULL) {
action = 'Q';
device = options.query;
}
if (options.unregister_dev != NULL) {
action = 'D';
device = options.unregister_dev;
}
if (options.targets != NULL) {
action = 's';
device = options.targets;
}
if (options.terminate != NULL) {
action = 'L';
target = options.terminate;
}
if (options.metadata) {
no_connect = true;
required_agent = true;
action = 'M';
}
if (options.reboot_host != NULL) {
no_connect = true;
action = 'B';
target = options.reboot_host;
crm_log_args(argc, argv);
}
if (options.fence_host != NULL) {
no_connect = true;
action = 'F';
target = options.fence_host;
crm_log_args(argc, argv);
}
if (options.unfence_host != NULL) {
no_connect = true;
action = 'U';
target = options.unfence_host;
crm_log_args(argc, argv);
}
if (options.confirm_host != NULL) {
action = 'C';
target = options.confirm_host;
crm_log_args(argc, argv);
}
if (options.last_fenced != NULL) {
action = 'h';
target = options.last_fenced;
}
if (options.history != NULL) {
action = 'H';
target = options.history;
}
if (options.register_level != NULL) {
action = 'r';
target = options.register_level;
}
if (options.unregister_level != NULL) {
action = 'd';
target = options.unregister_level;
}
if (optind > argc || action == 0) {
char *help = g_option_context_get_help(context, TRUE, NULL);
out->err(out, "%s", help);
g_free(help);
exit_code = CRM_EX_USAGE;
goto done;
}
if (required_agent && options.agent == NULL) {
char *help = g_option_context_get_help(context, TRUE, NULL);
out->err(out, "Please specify an agent to query using -a,--agent [value]");
out->err(out, "%s", help);
g_free(help);
exit_code = CRM_EX_USAGE;
goto done;
}
st = stonith_api_new();
if (st == NULL) {
rc = -ENOMEM;
} else if (!no_connect) {
rc = st->cmds->connect(st, name, NULL);
}
if (rc < 0) {
out->err(out, "Could not connect to fencer: %s", pcmk_strerror(rc));
exit_code = CRM_EX_DISCONNECT;
goto done;
}
switch (action) {
case 'I':
rc = pcmk__fence_installed(out, st, options.timeout*1000);
if (rc != pcmk_rc_ok) {
out->err(out, "Failed to list installed devices: %s", pcmk_strerror(rc));
}
break;
case 'L':
rc = pcmk__fence_registered(out, st, target, options.timeout*1000);
if (rc != pcmk_rc_ok) {
out->err(out, "Failed to list registered devices: %s", pcmk_strerror(rc));
}
break;
case 'Q':
rc = st->cmds->monitor(st, st_opts, device, options.timeout);
if (rc != pcmk_rc_ok) {
rc = st->cmds->list(st, st_opts, device, NULL, options.timeout);
}
rc = pcmk_legacy2rc(rc);
break;
case 's':
rc = pcmk__fence_list_targets(out, st, target, options.timeout*1000);
if (rc != pcmk_rc_ok) {
out->err(out, "Couldn't list targets: %s", pcmk_strerror(rc));
}
break;
case 'R':
rc = st->cmds->register_device(st, st_opts, device, NULL, options.agent,
options.params);
rc = pcmk_legacy2rc(rc);
break;
case 'D':
rc = st->cmds->remove_device(st, st_opts, device);
rc = pcmk_legacy2rc(rc);
break;
case 'd':
rc = pcmk__fence_unregister_level(st, target, options.fence_level);
break;
case 'r':
rc = pcmk__fence_register_level(st, target, options.fence_level, options.devices);
break;
case 'M':
rc = pcmk__fence_metadata(out, st, options.agent, options.timeout*1000);
if (rc != pcmk_rc_ok) {
out->err(out, "Can't get fence agent meta-data: %s", pcmk_strerror(rc));
}
break;
case 'C':
rc = st->cmds->confirm(st, st_opts, target);
rc = pcmk_legacy2rc(rc);
break;
case 'B':
rc = pcmk__fence_action(st, target, "reboot", name, options.timeout*1000,
options.tolerance*1000, options.delay);
break;
case 'F':
rc = pcmk__fence_action(st, target, "off", name, options.timeout*1000,
options.tolerance*1000, options.delay);
break;
case 'U':
rc = pcmk__fence_action(st, target, "on", name, options.timeout*1000,
options.tolerance*1000, options.delay);
break;
case 'h':
rc = pcmk__fence_last(out, target, options.as_nodeid);
break;
case 'H':
rc = pcmk__fence_history(out, st, target, options.timeout*1000, args->quiet,
args->verbosity, options.broadcast, options.cleanup);
break;
case 'K':
device = options.devices ? options.devices->key : NULL;
rc = pcmk__fence_validate(out, st, options.agent, device, options.params,
options.timeout*1000);
break;
}
crm_info("Command returned: %s (%d)", pcmk_rc_str(rc), rc);
exit_code = pcmk_rc2exitc(rc);
done:
g_strfreev(processed_args);
g_clear_error(&error);
pcmk__free_arg_context(context);
if (out != NULL) {
out->finish(out, exit_code, true, NULL);
pcmk__output_free(out);
}
free(name);
stonith_key_value_freeall(options.params, 1, 1);
if (st != NULL) {
st->cmds->disconnect(st);
stonith_api_delete(st);
}
return exit_code;
}