diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h index f3d38a746a..7d609345ff 100644 --- a/include/crm/fencing/internal.h +++ b/include/crm/fencing/internal.h @@ -1,153 +1,155 @@ /* * Copyright 2011-2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef STONITH_NG_INTERNAL__H # define STONITH_NG_INTERNAL__H # include # include # include # include struct stonith_action_s; typedef struct stonith_action_s stonith_action_t; stonith_action_t *stonith_action_create(const char *agent, const char *_action, const char *victim, uint32_t victim_nodeid, int timeout, GHashTable * device_args, GHashTable * port_map); void stonith__destroy_action(stonith_action_t *action); void stonith__action_result(stonith_action_t *action, int *rc, char **output, char **error_output); int stonith_action_execute_async(stonith_action_t * action, void *userdata, void (*done) (GPid pid, int rc, const char *output, gpointer user_data), void (*fork_cb) (GPid pid, gpointer user_data)); int stonith__execute(stonith_action_t *action); xmlNode *create_level_registration_xml(const char *node, const char *pattern, const char *attr, const char *value, int level, stonith_key_value_t *device_list); xmlNode *create_device_registration_xml(const char *id, enum stonith_namespace namespace, const char *agent, stonith_key_value_t *params, const char *rsc_provides); void stonith__register_messages(pcmk__output_t *out); GList *stonith__parse_targets(const char *hosts); +gboolean stonith__later_succeeded(stonith_history_t *event, stonith_history_t *top_history); + # define ST_LEVEL_MAX 10 # define F_STONITH_CLIENTID "st_clientid" # define F_STONITH_CALLOPTS "st_callopt" # define F_STONITH_CALLID "st_callid" # define F_STONITH_CALLDATA "st_calldata" # define F_STONITH_OPERATION "st_op" # define F_STONITH_TARGET "st_target" # define F_STONITH_REMOTE_OP_ID "st_remote_op" # define F_STONITH_RC "st_rc" /*! Timeout period per a device execution */ # define F_STONITH_TIMEOUT "st_timeout" # define F_STONITH_TOLERANCE "st_tolerance" /*! Action specific timeout period returned in query of fencing devices. */ # define F_STONITH_ACTION_TIMEOUT "st_action_timeout" /*! Host in query result is not allowed to run this action */ # define F_STONITH_ACTION_DISALLOWED "st_action_disallowed" /*! Maximum of random fencing delay for a device */ # define F_STONITH_DELAY_MAX "st_delay_max" /*! Base delay used for a fencing delay */ # define F_STONITH_DELAY_BASE "st_delay_base" /*! Has this device been verified using a monitor type * operation (monitor, list, status) */ # define F_STONITH_DEVICE_VERIFIED "st_monitor_verified" /*! device is required for this action */ # define F_STONITH_DEVICE_REQUIRED "st_required" /*! number of available devices in query result */ # define F_STONITH_AVAILABLE_DEVICES "st-available-devices" # define F_STONITH_CALLBACK_TOKEN "st_async_id" # define F_STONITH_CLIENTNAME "st_clientname" # define F_STONITH_CLIENTNODE "st_clientnode" # define F_STONITH_NOTIFY_ACTIVATE "st_notify_activate" # define F_STONITH_NOTIFY_DEACTIVATE "st_notify_deactivate" # define F_STONITH_DELEGATE "st_delegate" /*! The node initiating the stonith operation. If an operation * is relayed, this is the last node the operation lands on. When * in standalone mode, origin is the client's id that originated the * operation. */ # define F_STONITH_ORIGIN "st_origin" # define F_STONITH_HISTORY_LIST "st_history" # define F_STONITH_DATE "st_date" # define F_STONITH_STATE "st_state" # define F_STONITH_ACTIVE "st_active" # define F_STONITH_DIFFERENTIAL "st_differential" # define F_STONITH_DEVICE "st_device_id" # define F_STONITH_ACTION "st_device_action" # define F_STONITH_MODE "st_mode" # define T_STONITH_NG "stonith-ng" # define T_STONITH_REPLY "st-reply" /*! For async operations, an event from the server containing * the total amount of time the server is allowing for the operation * to take place is returned to the client. */ # define T_STONITH_TIMEOUT_VALUE "st-async-timeout-value" # define T_STONITH_NOTIFY "st_notify" # define STONITH_ATTR_HOSTARG "pcmk_host_argument" # define STONITH_ATTR_HOSTMAP "pcmk_host_map" # define STONITH_ATTR_HOSTLIST "pcmk_host_list" # define STONITH_ATTR_HOSTCHECK "pcmk_host_check" # define STONITH_ATTR_DELAY_MAX "pcmk_delay_max" # define STONITH_ATTR_DELAY_BASE "pcmk_delay_base" # define STONITH_ATTR_ACTION_LIMIT "pcmk_action_limit" # define STONITH_ATTR_ACTION_OP "action" # define STONITH_OP_EXEC "st_execute" # define STONITH_OP_TIMEOUT_UPDATE "st_timeout_update" # define STONITH_OP_QUERY "st_query" # define STONITH_OP_FENCE "st_fence" # define STONITH_OP_RELAY "st_relay" # define STONITH_OP_DEVICE_ADD "st_device_register" # define STONITH_OP_DEVICE_DEL "st_device_remove" # define STONITH_OP_FENCE_HISTORY "st_fence_history" # define STONITH_OP_LEVEL_ADD "st_level_add" # define STONITH_OP_LEVEL_DEL "st_level_remove" # define STONITH_WATCHDOG_AGENT "#watchdog" # ifdef HAVE_STONITH_STONITH_H // utilities from st_lha.c int stonith__list_lha_agents(stonith_key_value_t **devices); int stonith__lha_metadata(const char *agent, int timeout, char **output); bool stonith__agent_is_lha(const char *agent); int stonith__lha_validate(stonith_t *st, int call_options, const char *target, const char *agent, GHashTable *params, int timeout, char **output, char **error_output); # endif // utilities from st_rhcs.c int stonith__list_rhcs_agents(stonith_key_value_t **devices); int stonith__rhcs_metadata(const char *agent, int timeout, char **output); bool stonith__agent_is_rhcs(const char *agent); int stonith__rhcs_validate(stonith_t *st, int call_options, const char *target, const char *agent, GHashTable *params, int timeout, char **output, char **error_output); #endif diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index c265eeae0c..3b4e7f0e88 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1,2515 +1,2541 @@ /* * Copyright 2004-2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if SUPPORT_CIBSECRETS # include #endif CRM_TRACE_INIT_DATA(stonith); struct stonith_action_s { /*! user defined data */ char *agent; char *action; char *victim; GHashTable *args; int timeout; int async; void *userdata; void (*done_cb) (GPid pid, gint status, const char *output, gpointer user_data); void (*fork_cb) (GPid pid, gpointer user_data); svc_action_t *svc_action; /*! internal timing information */ time_t initial_start_time; int tries; int remaining_timeout; int max_retries; /* device output data */ GPid pid; int rc; char *output; char *error; }; typedef struct stonith_private_s { char *token; crm_ipc_t *ipc; mainloop_io_t *source; GHashTable *stonith_op_callback_table; GList *notify_list; int notify_refcnt; bool notify_deletes; void (*op_callback) (stonith_t * st, stonith_callback_data_t * data); } stonith_private_t; typedef struct stonith_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*notify) (stonith_t * st, stonith_event_t * e); bool delete; } stonith_notify_client_t; typedef struct stonith_callback_client_s { void (*callback) (stonith_t * st, stonith_callback_data_t * data); const char *id; void *user_data; gboolean only_success; gboolean allow_timeout_updates; struct timer_rec_s *timer; } stonith_callback_client_t; struct notify_blob_s { stonith_t *stonith; xmlNode *xml; }; struct timer_rec_s { int call_id; int timeout; guint ref; stonith_t *stonith; }; typedef int (*stonith_op_t) (const char *, int, const char *, xmlNode *, xmlNode *, xmlNode *, xmlNode **, xmlNode **); bool stonith_dispatch(stonith_t * st); xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options); static int stonith_send_command(stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options, int timeout); static void stonith_connection_destroy(gpointer user_data); static void stonith_send_notification(gpointer data, gpointer user_data); static int internal_stonith_action_execute(stonith_action_t * action); static void log_action(stonith_action_t *action, pid_t pid); /*! * \brief Get agent namespace by name * * \param[in] namespace_s Name of namespace as string * * \return Namespace as enum value */ enum stonith_namespace stonith_text2namespace(const char *namespace_s) { if ((namespace_s == NULL) || !strcmp(namespace_s, "any")) { return st_namespace_any; } else if (!strcmp(namespace_s, "redhat") || !strcmp(namespace_s, "stonith-ng")) { return st_namespace_rhcs; } else if (!strcmp(namespace_s, "internal")) { return st_namespace_internal; } else if (!strcmp(namespace_s, "heartbeat")) { return st_namespace_lha; } return st_namespace_invalid; } /*! * \brief Get agent namespace name * * \param[in] namespace Namespace as enum value * * \return Namespace name as string */ const char * stonith_namespace2text(enum stonith_namespace st_namespace) { switch (st_namespace) { case st_namespace_any: return "any"; case st_namespace_rhcs: return "stonith-ng"; case st_namespace_internal: return "internal"; case st_namespace_lha: return "heartbeat"; default: break; } return "unsupported"; } /*! * \brief Determine namespace of a fence agent * * \param[in] agent Fence agent type * \param[in] namespace_s Name of agent namespace as string, if known * * \return Namespace of specified agent, as enum value */ enum stonith_namespace stonith_get_namespace(const char *agent, const char *namespace_s) { if (safe_str_eq(namespace_s, "internal")) { return st_namespace_internal; } if (stonith__agent_is_rhcs(agent)) { return st_namespace_rhcs; } #if HAVE_STONITH_STONITH_H if (stonith__agent_is_lha(agent)) { return st_namespace_lha; } #endif crm_err("Unknown fence agent: %s", agent); return st_namespace_invalid; } static void log_action(stonith_action_t *action, pid_t pid) { if (action->output) { /* Logging the whole string confuses syslog when the string is xml */ char *prefix = crm_strdup_printf("%s[%d] stdout:", action->agent, pid); crm_log_output(LOG_TRACE, prefix, action->output); free(prefix); } if (action->error) { /* Logging the whole string confuses syslog when the string is xml */ char *prefix = crm_strdup_printf("%s[%d] stderr:", action->agent, pid); crm_log_output(LOG_WARNING, prefix, action->error); free(prefix); } } /* when cycling through the list we don't want to delete items so just mark them and when we know nobody is using the list loop over it to remove the marked items */ static void foreach_notify_entry (stonith_private_t *private, GFunc func, gpointer user_data) { private->notify_refcnt++; g_list_foreach(private->notify_list, func, user_data); private->notify_refcnt--; if ((private->notify_refcnt == 0) && private->notify_deletes) { GList *list_item = private->notify_list; private->notify_deletes = FALSE; while (list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; GList *next = g_list_next(list_item); if (list_client->delete) { free(list_client); private->notify_list = g_list_delete_link(private->notify_list, list_item); } list_item = next; } } } static void stonith_connection_destroy(gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *native = NULL; struct notify_blob_s blob; crm_trace("Sending destroyed notification"); blob.stonith = stonith; blob.xml = create_xml_node(NULL, "notify"); native = stonith->st_private; native->ipc = NULL; native->source = NULL; free(native->token); native->token = NULL; stonith->state = stonith_disconnected; crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); foreach_notify_entry(native, stonith_send_notification, &blob); free_xml(blob.xml); } xmlNode * create_device_registration_xml(const char *id, enum stonith_namespace namespace, const char *agent, stonith_key_value_t *params, const char *rsc_provides) { xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE); xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); #if HAVE_STONITH_STONITH_H if (namespace == st_namespace_any) { namespace = stonith_get_namespace(agent, NULL); } if (namespace == st_namespace_lha) { hash2field((gpointer) "plugin", (gpointer) agent, args); agent = "fence_legacy"; } #endif crm_xml_add(data, XML_ATTR_ID, id); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, "agent", agent); if ((namespace != st_namespace_any) && (namespace != st_namespace_invalid)) { crm_xml_add(data, "namespace", stonith_namespace2text(namespace)); } if (rsc_provides) { crm_xml_add(data, "rsc_provides", rsc_provides); } for (; params; params = params->next) { hash2field((gpointer) params->key, (gpointer) params->value, args); } return data; } static int stonith_api_register_device(stonith_t * st, int call_options, const char *id, const char *namespace, const char *agent, stonith_key_value_t * params) { int rc = 0; xmlNode *data = NULL; data = create_device_registration_xml(id, stonith_text2namespace(namespace), agent, params, NULL); rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_remove_device(stonith_t * st, int call_options, const char *name) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, XML_ATTR_ID, name); rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_remove_level_full(stonith_t *st, int options, const char *node, const char *pattern, const char *attr, const char *value, int level) { int rc = 0; xmlNode *data = NULL; CRM_CHECK(node || pattern || (attr && value), return -EINVAL); data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); if (node) { crm_xml_add(data, XML_ATTR_STONITH_TARGET, node); } else if (pattern) { crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern); } else { crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr); crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value); } crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level); rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0); free_xml(data); return rc; } static int stonith_api_remove_level(stonith_t * st, int options, const char *node, int level) { return stonith_api_remove_level_full(st, options, node, NULL, NULL, NULL, level); } /*! * \internal * \brief Create XML for fence topology level registration request * * \param[in] node If not NULL, target level by this node name * \param[in] pattern If not NULL, target by node name using this regex * \param[in] attr If not NULL, target by this node attribute * \param[in] value If not NULL, target by this node attribute value * \param[in] level Index number of level to register * \param[in] device_list List of devices in level * * \return Newly allocated XML tree on success, NULL otherwise * * \note The caller should set only one of node, pattern or attr/value. */ xmlNode * create_level_registration_xml(const char *node, const char *pattern, const char *attr, const char *value, int level, stonith_key_value_t *device_list) { int len = 0; char *list = NULL; xmlNode *data; CRM_CHECK(node || pattern || (attr && value), return NULL); data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL); CRM_CHECK(data, return NULL); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add_int(data, XML_ATTR_ID, level); crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level); if (node) { crm_xml_add(data, XML_ATTR_STONITH_TARGET, node); } else if (pattern) { crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern); } else { crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr); crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value); } for (; device_list; device_list = device_list->next) { int adding = strlen(device_list->value); if(list) { adding++; /* +1 space */ } crm_trace("Adding %s (%dc) at offset %d", device_list->value, adding, len); list = realloc_safe(list, len + adding + 1); /* +1 EOS */ if (list == NULL) { crm_perror(LOG_CRIT, "Could not create device list"); free_xml(data); return NULL; } sprintf(list + len, "%s%s", len?",":"", device_list->value); len += adding; } crm_xml_add(data, XML_ATTR_STONITH_DEVICES, list); free(list); return data; } static int stonith_api_register_level_full(stonith_t * st, int options, const char *node, const char *pattern, const char *attr, const char *value, int level, stonith_key_value_t *device_list) { int rc = 0; xmlNode *data = create_level_registration_xml(node, pattern, attr, value, level, device_list); CRM_CHECK(data != NULL, return -EINVAL); rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0); free_xml(data); return rc; } static int stonith_api_register_level(stonith_t * st, int options, const char *node, int level, stonith_key_value_t * device_list) { return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL, level, device_list); } static void append_arg(const char *key, const char *value, GHashTable **args) { CRM_CHECK(key != NULL, return); CRM_CHECK(value != NULL, return); CRM_CHECK(args != NULL, return); if (strstr(key, "pcmk_")) { return; } else if (strstr(key, CRM_META)) { return; } else if (safe_str_eq(key, "crm_feature_set")) { return; } if (!*args) { *args = crm_str_table_new(); } CRM_CHECK(*args != NULL, return); crm_trace("Appending: %s=%s", key, value); g_hash_table_replace(*args, strdup(key), strdup(value)); } static void append_config_arg(gpointer key, gpointer value, gpointer user_data) { /* The fencer will filter action out when it registers the device, * but ignore it here just in case any other library callers * fail to do so. */ if (safe_str_neq(key, STONITH_ATTR_ACTION_OP)) { append_arg(key, value, user_data); return; } } static GHashTable * make_args(const char *agent, const char *action, const char *victim, uint32_t victim_nodeid, GHashTable * device_args, GHashTable * port_map) { char buffer[512]; GHashTable *arg_list = NULL; const char *value = NULL; CRM_CHECK(action != NULL, return NULL); snprintf(buffer, sizeof(buffer), "pcmk_%s_action", action); if (device_args) { value = g_hash_table_lookup(device_args, buffer); } if (value) { crm_debug("Substituting action '%s' for requested operation '%s'", value, action); action = value; } append_arg(STONITH_ATTR_ACTION_OP, action, &arg_list); if (victim && device_args) { const char *alias = victim; const char *param = g_hash_table_lookup(device_args, STONITH_ATTR_HOSTARG); if (port_map && g_hash_table_lookup(port_map, victim)) { alias = g_hash_table_lookup(port_map, victim); } /* Always supply the node's name, too: * https://github.com/ClusterLabs/fence-agents/blob/master/doc/FenceAgentAPI.md */ append_arg("nodename", victim, &arg_list); if (victim_nodeid) { char nodeid_str[33] = { 0, }; if (snprintf(nodeid_str, 33, "%u", (unsigned int)victim_nodeid)) { crm_info("For stonith action (%s) for victim %s, adding nodeid (%s) to parameters", action, victim, nodeid_str); append_arg("nodeid", nodeid_str, &arg_list); } } /* Check if we need to supply the victim in any other form */ if(safe_str_eq(agent, "fence_legacy")) { value = agent; } else if (param == NULL) { param = "port"; value = g_hash_table_lookup(device_args, param); } else if (safe_str_eq(param, "none")) { value = param; /* Nothing more to do */ } else { value = g_hash_table_lookup(device_args, param); } /* Don't overwrite explictly set values for $param */ if (value == NULL || safe_str_eq(value, "dynamic")) { crm_debug("Performing %s action for node '%s' as '%s=%s'", action, victim, param, alias); append_arg(param, alias, &arg_list); } } if (device_args) { g_hash_table_foreach(device_args, append_config_arg, &arg_list); } return arg_list; } /*! * \internal * \brief Free all memory used by a stonith action * * \param[in,out] action Action to free */ void stonith__destroy_action(stonith_action_t *action) { if (action) { free(action->agent); if (action->args) { g_hash_table_destroy(action->args); } free(action->action); free(action->victim); if (action->svc_action) { services_action_free(action->svc_action); } free(action->output); free(action->error); free(action); } } /*! * \internal * \brief Get the result of an executed stonith action * * \param[in,out] action Executed action * \param[out] rc Where to store result code (or NULL) * \param[out] output Where to store standard output (or NULL) * \param[out] error_output Where to store standard error output (or NULL) * * \note If output or error_output is not NULL, the caller is responsible for * freeing the memory. */ void stonith__action_result(stonith_action_t *action, int *rc, char **output, char **error_output) { if (rc) { *rc = pcmk_ok; } if (output) { *output = NULL; } if (error_output) { *error_output = NULL; } if (action != NULL) { if (rc) { *rc = action->rc; } if (output && action->output) { *output = action->output; action->output = NULL; // hand off memory management to caller } if (error_output && action->error) { *error_output = action->error; action->error = NULL; // hand off memory management to caller } } } #define FAILURE_MAX_RETRIES 2 stonith_action_t * stonith_action_create(const char *agent, const char *_action, const char *victim, uint32_t victim_nodeid, int timeout, GHashTable * device_args, GHashTable * port_map) { stonith_action_t *action; action = calloc(1, sizeof(stonith_action_t)); action->args = make_args(agent, _action, victim, victim_nodeid, device_args, port_map); crm_debug("Preparing '%s' action for %s using agent %s", _action, (victim? victim : "no target"), agent); action->agent = strdup(agent); action->action = strdup(_action); if (victim) { action->victim = strdup(victim); } action->timeout = action->remaining_timeout = timeout; action->max_retries = FAILURE_MAX_RETRIES; if (device_args) { char buffer[512]; const char *value = NULL; snprintf(buffer, sizeof(buffer), "pcmk_%s_retries", _action); value = g_hash_table_lookup(device_args, buffer); if (value) { action->max_retries = atoi(value); } } return action; } static gboolean update_remaining_timeout(stonith_action_t * action) { int diff = time(NULL) - action->initial_start_time; if (action->tries >= action->max_retries) { crm_info("Attempted to execute agent %s (%s) the maximum number of times (%d) allowed", action->agent, action->action, action->max_retries); action->remaining_timeout = 0; } else if ((action->rc != -ETIME) && diff < (action->timeout * 0.7)) { /* only set remaining timeout period if there is 30% * or greater of the original timeout period left */ action->remaining_timeout = action->timeout - diff; } else { action->remaining_timeout = 0; } return action->remaining_timeout ? TRUE : FALSE; } static int svc_action_to_errno(svc_action_t *svc_action) { int rv = pcmk_ok; if (svc_action->rc > 0) { /* Try to provide a useful error code based on the fence agent's * error output. */ if (svc_action->rc == PCMK_OCF_TIMEOUT) { rv = -ETIME; } else if (svc_action->stderr_data == NULL) { rv = -ENODATA; } else if (strstr(svc_action->stderr_data, "imed out")) { /* Some agents have their own internal timeouts */ rv = -ETIME; } else if (strstr(svc_action->stderr_data, "Unrecognised action")) { rv = -EOPNOTSUPP; } else { rv = -pcmk_err_generic; } } return rv; } static void stonith_action_async_done(svc_action_t *svc_action) { stonith_action_t *action = (stonith_action_t *) svc_action->cb_data; action->rc = svc_action_to_errno(svc_action); action->output = svc_action->stdout_data; svc_action->stdout_data = NULL; action->error = svc_action->stderr_data; svc_action->stderr_data = NULL; svc_action->params = NULL; crm_debug("Child process %d performing action '%s' exited with rc %d", action->pid, action->action, svc_action->rc); log_action(action, action->pid); if (action->rc != pcmk_ok && update_remaining_timeout(action)) { int rc = internal_stonith_action_execute(action); if (rc == pcmk_ok) { return; } } if (action->done_cb) { action->done_cb(action->pid, action->rc, action->output, action->userdata); } action->svc_action = NULL; // don't remove our caller stonith__destroy_action(action); } static void stonith_action_async_forked(svc_action_t *svc_action) { stonith_action_t *action = (stonith_action_t *) svc_action->cb_data; action->pid = svc_action->pid; action->svc_action = svc_action; if (action->fork_cb) { (action->fork_cb) (svc_action->pid, action->userdata); } crm_trace("Child process %d performing action '%s' successfully forked", action->pid, action->action); } static int internal_stonith_action_execute(stonith_action_t * action) { int rc = -EPROTO; int is_retry = 0; svc_action_t *svc_action = NULL; static int stonith_sequence = 0; char *buffer = NULL; if (!action->tries) { action->initial_start_time = time(NULL); } action->tries++; if (action->tries > 1) { crm_info("Attempt %d to execute %s (%s). remaining timeout is %d", action->tries, action->agent, action->action, action->remaining_timeout); is_retry = 1; } if (action->args == NULL || action->agent == NULL) goto fail; buffer = crm_strdup_printf(RH_STONITH_DIR "/%s", basename(action->agent)); svc_action = services_action_create_generic(buffer, NULL); free(buffer); svc_action->timeout = 1000 * action->remaining_timeout; svc_action->standard = strdup(PCMK_RESOURCE_CLASS_STONITH); svc_action->id = crm_strdup_printf("%s_%s_%d", basename(action->agent), action->action, action->tries); svc_action->agent = strdup(action->agent); svc_action->sequence = stonith_sequence++; svc_action->params = action->args; svc_action->cb_data = (void *) action; /* keep retries from executing out of control and free previous results */ if (is_retry) { free(action->output); action->output = NULL; free(action->error); action->error = NULL; sleep(1); } if (action->async) { /* async */ if(services_action_async_fork_notify(svc_action, &stonith_action_async_done, &stonith_action_async_forked) == FALSE) { services_action_free(svc_action); svc_action = NULL; } else { rc = 0; } } else { /* sync */ if (services_action_sync(svc_action)) { rc = 0; action->rc = svc_action_to_errno(svc_action); action->output = svc_action->stdout_data; svc_action->stdout_data = NULL; action->error = svc_action->stderr_data; svc_action->stderr_data = NULL; } else { action->rc = -ECONNABORTED; rc = action->rc; } svc_action->params = NULL; services_action_free(svc_action); } fail: return rc; } /*! * \internal * \brief Kick off execution of an async stonith action * * \param[in,out] action Action to be executed * \param[in,out] userdata Datapointer to be passed to callbacks * \param[in] done Callback to notify action has failed/succeeded * \param[in] fork_callback Callback to notify successful fork of child * * \return pcmk_ok if ownership of action has been taken, -errno otherwise */ int stonith_action_execute_async(stonith_action_t * action, void *userdata, void (*done) (GPid pid, int rc, const char *output, gpointer user_data), void (*fork_cb) (GPid pid, gpointer user_data)) { if (!action) { return -EINVAL; } action->userdata = userdata; action->done_cb = done; action->fork_cb = fork_cb; action->async = 1; return internal_stonith_action_execute(action); } /*! * \internal * \brief Execute a stonith action * * \param[in,out] action Action to execute * * \return pcmk_ok on success, -errno otherwise */ int stonith__execute(stonith_action_t *action) { int rc = pcmk_ok; CRM_CHECK(action != NULL, return -EINVAL); // Keep trying until success, max retries, or timeout do { rc = internal_stonith_action_execute(action); } while ((rc != pcmk_ok) && update_remaining_timeout(action)); return rc; } static int stonith_api_device_list(stonith_t * stonith, int call_options, const char *namespace, stonith_key_value_t ** devices, int timeout) { int count = 0; enum stonith_namespace ns = stonith_text2namespace(namespace); if (devices == NULL) { crm_err("Parameter error: stonith_api_device_list"); return -EFAULT; } #if HAVE_STONITH_STONITH_H // Include Linux-HA agents if requested if ((ns == st_namespace_any) || (ns == st_namespace_lha)) { count += stonith__list_lha_agents(devices); } #endif // Include Red Hat agents if requested if ((ns == st_namespace_any) || (ns == st_namespace_rhcs)) { count += stonith__list_rhcs_agents(devices); } return count; } static int stonith_api_device_metadata(stonith_t * stonith, int call_options, const char *agent, const char *namespace, char **output, int timeout) { /* By executing meta-data directly, we can get it from stonith_admin when * the cluster is not running, which is important for higher-level tools. */ enum stonith_namespace ns = stonith_get_namespace(agent, namespace); crm_trace("Looking up metadata for %s agent %s", stonith_namespace2text(ns), agent); switch (ns) { case st_namespace_rhcs: return stonith__rhcs_metadata(agent, timeout, output); #if HAVE_STONITH_STONITH_H case st_namespace_lha: return stonith__lha_metadata(agent, timeout, output); #endif default: errno = EINVAL; crm_perror(LOG_ERR, "Agent %s not found or does not support meta-data", agent); break; } return -EINVAL; } static int stonith_api_query(stonith_t * stonith, int call_options, const char *target, stonith_key_value_t ** devices, int timeout) { int rc = 0, lpc = 0, max = 0; xmlNode *data = NULL; xmlNode *output = NULL; xmlXPathObjectPtr xpathObj = NULL; CRM_CHECK(devices != NULL, return -EINVAL); data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, target); crm_xml_add(data, F_STONITH_ACTION, "off"); rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout); if (rc < 0) { return rc; } xpathObj = xpath_search(output, "//@agent"); if (xpathObj) { max = numXpathResults(xpathObj); for (lpc = 0; lpc < max; lpc++) { xmlNode *match = getXpathResult(xpathObj, lpc); CRM_LOG_ASSERT(match != NULL); if(match != NULL) { xmlChar *match_path = xmlGetNodePath(match); crm_info("%s[%d] = %s", "//@agent", lpc, match_path); free(match_path); *devices = stonith_key_value_add(*devices, NULL, crm_element_value(match, XML_ATTR_ID)); } } freeXpathObject(xpathObj); } free_xml(output); free_xml(data); return max; } static int stonith_api_call(stonith_t * stonith, int call_options, const char *id, const char *action, const char *victim, int timeout, xmlNode ** output) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, F_STONITH_DEVICE, id); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add(data, F_STONITH_TARGET, victim); rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output, call_options, timeout); free_xml(data); return rc; } static int stonith_api_list(stonith_t * stonith, int call_options, const char *id, char **list_info, int timeout) { int rc; xmlNode *output = NULL; rc = stonith_api_call(stonith, call_options, id, "list", NULL, timeout, &output); if (output && list_info) { const char *list_str; list_str = crm_element_value(output, "st_output"); if (list_str) { *list_info = strdup(list_str); } } if (output) { free_xml(output); } return rc; } static int stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout) { return stonith_api_call(stonith, call_options, id, "monitor", NULL, timeout, NULL); } static int stonith_api_status(stonith_t * stonith, int call_options, const char *id, const char *port, int timeout) { return stonith_api_call(stonith, call_options, id, "status", port, timeout, NULL); } static int stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, int timeout, int tolerance) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); return rc; } static int stonith_api_confirm(stonith_t * stonith, int call_options, const char *target) { return stonith_api_fence(stonith, call_options | st_opt_manual_ack, target, "off", 0, 0); } static int stonith_api_history(stonith_t * stonith, int call_options, const char *node, stonith_history_t ** history, int timeout) { int rc = 0; xmlNode *data = NULL; xmlNode *output = NULL; stonith_history_t *last = NULL; *history = NULL; if (node) { data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); } rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output, call_options | st_opt_sync_call, timeout); free_xml(data); if (rc == 0) { xmlNode *op = NULL; xmlNode *reply = get_xpath_object("//" F_STONITH_HISTORY_LIST, output, LOG_TRACE); for (op = __xml_first_child(reply); op != NULL; op = __xml_next(op)) { stonith_history_t *kvp; int completed; kvp = calloc(1, sizeof(stonith_history_t)); kvp->target = crm_element_value_copy(op, F_STONITH_TARGET); kvp->action = crm_element_value_copy(op, F_STONITH_ACTION); kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN); kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE); kvp->client = crm_element_value_copy(op, F_STONITH_CLIENTNAME); crm_element_value_int(op, F_STONITH_DATE, &completed); kvp->completed = (time_t) completed; crm_element_value_int(op, F_STONITH_STATE, &kvp->state); if (last) { last->next = kvp; } else { *history = kvp; } last = kvp; } } free_xml(output); return rc; } void stonith_history_free(stonith_history_t *history) { stonith_history_t *hp, *hp_old; for (hp = history; hp; hp_old = hp, hp = hp->next, free(hp_old)) { free(hp->target); free(hp->action); free(hp->origin); free(hp->delegate); free(hp->client); } } /*! * \brief Deprecated (use stonith_get_namespace() instead) */ const char * get_stonith_provider(const char *agent, const char *provider) { return stonith_namespace2text(stonith_get_namespace(agent, provider)); } static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const stonith_notify_client_t *a_client = a; const stonith_notify_client_t *b_client = b; if (a_client->delete || b_client->delete) { /* make entries marked for deletion not findable */ return -1; } CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if (rc == 0) { if (a_client->notify == NULL || b_client->notify == NULL) { return 0; } else if (a_client->notify == b_client->notify) { return 0; } else if (((long)a_client->notify) < ((long)b_client->notify)) { crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return -1; } crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return 1; } return rc; } xmlNode * stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options) { xmlNode *op_msg = create_xml_node(NULL, "stonith_command"); CRM_CHECK(op_msg != NULL, return NULL); CRM_CHECK(token != NULL, return NULL); crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command"); crm_xml_add(op_msg, F_TYPE, T_STONITH_NG); crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token); crm_xml_add(op_msg, F_STONITH_OPERATION, op); crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id); crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options); crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options); if (data != NULL) { add_message_xml(op_msg, F_STONITH_CALLDATA, data); } return op_msg; } static void stonith_destroy_op_callback(gpointer data) { stonith_callback_client_t *blob = data; if (blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } free(blob->timer); free(blob); } static int stonith_api_signoff(stonith_t * stonith) { stonith_private_t *native = stonith->st_private; crm_debug("Disconnecting from the fencer"); if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if (native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } free(native->token); native->token = NULL; stonith->state = stonith_disconnected; return pcmk_ok; } static int stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks) { stonith_private_t *private = stonith->st_private; if (all_callbacks) { private->op_callback = NULL; g_hash_table_destroy(private->stonith_op_callback_table); private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); } else if (call_id == 0) { private->op_callback = NULL; } else { g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); } return pcmk_ok; } static void invoke_callback(stonith_t * st, int call_id, int rc, void *userdata, void (*callback) (stonith_t * st, stonith_callback_data_t * data)) { stonith_callback_data_t data = { 0, }; data.call_id = call_id; data.rc = rc; data.userdata = userdata; callback(st, &data); } static void stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc) { stonith_private_t *private = NULL; stonith_callback_client_t *blob = NULL; stonith_callback_client_t local_blob; CRM_CHECK(stonith != NULL, return); CRM_CHECK(stonith->st_private != NULL, return); private = stonith->st_private; local_blob.id = NULL; local_blob.callback = NULL; local_blob.user_data = NULL; local_blob.only_success = FALSE; if (msg != NULL) { crm_element_value_int(msg, F_STONITH_RC, &rc); crm_element_value_int(msg, F_STONITH_CALLID, &call_id); } CRM_CHECK(call_id > 0, crm_log_xml_err(msg, "Bad result")); blob = g_hash_table_lookup(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); if (blob != NULL) { local_blob = *blob; blob = NULL; stonith_api_del_callback(stonith, call_id, FALSE); } else { crm_trace("No callback found for call %d", call_id); local_blob.callback = NULL; } if (local_blob.callback != NULL && (rc == pcmk_ok || local_blob.only_success == FALSE)) { crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id); invoke_callback(stonith, call_id, rc, local_blob.user_data, local_blob.callback); } else if (private->op_callback == NULL && rc != pcmk_ok) { crm_warn("Fencing command failed: %s", pcmk_strerror(rc)); crm_log_xml_debug(msg, "Failed fence update"); } if (private->op_callback != NULL) { crm_trace("Invoking global callback for call %d", call_id); invoke_callback(stonith, call_id, rc, NULL, private->op_callback); } crm_trace("OP callback activated."); } static gboolean stonith_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout); stonith_perform_callback(timer->stonith, NULL, timer->call_id, -ETIME); /* Always return TRUE, never remove the handler * We do that in stonith_del_callback() */ return TRUE; } static void set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id, int timeout) { struct timer_rec_s *async_timer = callback->timer; if (timeout <= 0) { return; } if (!async_timer) { async_timer = calloc(1, sizeof(struct timer_rec_s)); callback->timer = async_timer; } async_timer->stonith = stonith; async_timer->call_id = call_id; /* Allow a fair bit of grace to allow the server to tell us of a timeout * This is only a fallback */ async_timer->timeout = (timeout + 60) * 1000; if (async_timer->ref) { g_source_remove(async_timer->ref); } async_timer->ref = g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer); } static void update_callback_timeout(int call_id, int timeout, stonith_t * st) { stonith_callback_client_t *callback = NULL; stonith_private_t *private = st->st_private; callback = g_hash_table_lookup(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); if (!callback || !callback->allow_timeout_updates) { return; } set_callback_timeout(callback, st, call_id, timeout); } static int stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) { const char *type = NULL; struct notify_blob_s blob; stonith_t *st = userdata; stonith_private_t *private = NULL; CRM_ASSERT(st != NULL); private = st->st_private; blob.stonith = st; blob.xml = string2xml(buffer); if (blob.xml == NULL) { crm_warn("Received malformed message from fencer: %s", buffer); return 0; } /* do callbacks */ type = crm_element_value(blob.xml, F_TYPE); crm_trace("Activating %s callbacks...", type); if (safe_str_eq(type, T_STONITH_NG)) { stonith_perform_callback(st, blob.xml, 0, 0); } else if (safe_str_eq(type, T_STONITH_NOTIFY)) { foreach_notify_entry(private, stonith_send_notification, &blob); } else if (safe_str_eq(type, T_STONITH_TIMEOUT_VALUE)) { int call_id = 0; int timeout = 0; crm_element_value_int(blob.xml, F_STONITH_TIMEOUT, &timeout); crm_element_value_int(blob.xml, F_STONITH_CALLID, &call_id); update_callback_timeout(call_id, timeout, st); } else { crm_err("Unknown message type: %s", type); crm_log_xml_warn(blob.xml, "BadReply"); } free_xml(blob.xml); return 1; } static int stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) { int rc = pcmk_ok; stonith_private_t *native = NULL; const char *display_name = name? name : "client"; static struct ipc_client_callbacks st_callbacks = { .dispatch = stonith_dispatch_internal, .destroy = stonith_connection_destroy }; CRM_CHECK(stonith != NULL, return -EINVAL); native = stonith->st_private; CRM_ASSERT(native != NULL); crm_debug("Attempting fencer connection by %s with%s mainloop", display_name, (stonith_fd? "out" : "")); stonith->state = stonith_connected_command; if (stonith_fd) { /* No mainloop */ native->ipc = crm_ipc_new("stonith-ng", 0); if (native->ipc && crm_ipc_connect(native->ipc)) { *stonith_fd = crm_ipc_get_fd(native->ipc); } else if (native->ipc) { crm_ipc_close(native->ipc); crm_ipc_destroy(native->ipc); native->ipc = NULL; } } else { /* With mainloop */ native->source = mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks); native->ipc = mainloop_get_ipc_client(native->source); } if (native->ipc == NULL) { rc = -ENOTCONN; } else { xmlNode *reply = NULL; xmlNode *hello = create_xml_node(NULL, "stonith_command"); crm_xml_add(hello, F_TYPE, T_STONITH_NG); crm_xml_add(hello, F_STONITH_OPERATION, CRM_OP_REGISTER); crm_xml_add(hello, F_STONITH_CLIENTNAME, name); rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply); if (rc < 0) { crm_debug("Couldn't register with the fencer: %s " CRM_XS " rc=%d", pcmk_strerror(rc), rc); rc = -ECOMM; } else if (reply == NULL) { crm_debug("Couldn't register with the fencer: no reply"); rc = -EPROTO; } else { const char *msg_type = crm_element_value(reply, F_STONITH_OPERATION); native->token = crm_element_value_copy(reply, F_STONITH_CLIENTID); if (safe_str_neq(msg_type, CRM_OP_REGISTER)) { crm_debug("Couldn't register with the fencer: invalid reply type '%s'", (msg_type? msg_type : "(missing)")); crm_log_xml_debug(reply, "Invalid fencer reply"); rc = -EPROTO; } else if (native->token == NULL) { crm_debug("Couldn't register with the fencer: no token in reply"); crm_log_xml_debug(reply, "Invalid fencer reply"); rc = -EPROTO; } else { #if HAVE_MSGFROMIPC_TIMEOUT stonith->call_timeout = MAX_IPC_DELAY; #endif crm_debug("Connection to fencer by %s succeeded (registration token: %s)", display_name, native->token); rc = pcmk_ok; } } free_xml(reply); free_xml(hello); } if (rc != pcmk_ok) { crm_debug("Connection attempt to fencer by %s failed: %s " CRM_XS " rc=%d", display_name, pcmk_strerror(rc), rc); stonith->cmds->disconnect(stonith); } return rc; } static int stonith_set_notification(stonith_t * stonith, const char *callback, int enabled) { int rc = pcmk_ok; xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__); stonith_private_t *native = stonith->st_private; if (stonith->state != stonith_disconnected) { crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY); if (enabled) { crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback); } else { crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback); } rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL); if (rc < 0) { crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc); rc = -ECOMM; } else { rc = pcmk_ok; } } free_xml(notify_msg); return rc; } static int stonith_api_add_notification(stonith_t * stonith, const char *event, void (*callback) (stonith_t * stonith, stonith_event_t * e)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; private = stonith->st_private; crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list)); new_client = calloc(1, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = callback; list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc); if (list_item != NULL) { crm_warn("Callback already present"); free(new_client); return -ENOTUNIQ; } else { private->notify_list = g_list_append(private->notify_list, new_client); stonith_set_notification(stonith, event, 1); crm_trace("Callback added (%d)", g_list_length(private->notify_list)); } return pcmk_ok; } static int stonith_api_del_notification(stonith_t * stonith, const char *event) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; crm_debug("Removing callback for %s events", event); private = stonith->st_private; new_client = calloc(1, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = NULL; list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc); stonith_set_notification(stonith, event, 0); if (list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; if (private->notify_refcnt) { list_client->delete = TRUE; private->notify_deletes = TRUE; } else { private->notify_list = g_list_remove(private->notify_list, list_client); free(list_client); } crm_trace("Removed callback"); } else { crm_trace("Callback not present"); } free(new_client); return pcmk_ok; } static int stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options, void *user_data, const char *callback_name, void (*callback) (stonith_t * st, stonith_callback_data_t * data)) { stonith_callback_client_t *blob = NULL; stonith_private_t *private = NULL; CRM_CHECK(stonith != NULL, return -EINVAL); CRM_CHECK(stonith->st_private != NULL, return -EINVAL); private = stonith->st_private; if (call_id == 0) { private->op_callback = callback; } else if (call_id < 0) { if (!(options & st_opt_report_only_success)) { crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id)); invoke_callback(stonith, call_id, call_id, user_data, callback); } else { crm_warn("Fencer call failed: %s", pcmk_strerror(call_id)); } return FALSE; } blob = calloc(1, sizeof(stonith_callback_client_t)); blob->id = callback_name; blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE; blob->user_data = user_data; blob->callback = callback; blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE; if (timeout > 0) { set_callback_timeout(blob, stonith, call_id, timeout); } g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob); crm_trace("Added callback to %s for call %d", callback_name, call_id); return TRUE; } static void stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); stonith_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, crm_str(blob->id)); } void stonith_dump_pending_callbacks(stonith_t * stonith) { stonith_private_t *private = stonith->st_private; if (private->stonith_op_callback_table == NULL) { return; } return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL); } /* */ static stonith_event_t * xml_to_event(xmlNode * msg) { stonith_event_t *event = calloc(1, sizeof(stonith_event_t)); const char *ntype = crm_element_value(msg, F_SUBTYPE); char *data_addr = crm_strdup_printf("//%s", ntype); xmlNode *data = get_xpath_object(data_addr, msg, LOG_DEBUG); crm_log_xml_trace(msg, "stonith_notify"); crm_element_value_int(msg, F_STONITH_RC, &(event->result)); if (safe_str_eq(ntype, T_STONITH_NOTIFY_FENCE)) { event->operation = crm_element_value_copy(msg, F_STONITH_OPERATION); if (data) { event->origin = crm_element_value_copy(data, F_STONITH_ORIGIN); event->action = crm_element_value_copy(data, F_STONITH_ACTION); event->target = crm_element_value_copy(data, F_STONITH_TARGET); event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE); event->id = crm_element_value_copy(data, F_STONITH_REMOTE_OP_ID); event->client_origin = crm_element_value_copy(data, F_STONITH_CLIENTNAME); event->device = crm_element_value_copy(data, F_STONITH_DEVICE); } else { crm_err("No data for %s event", ntype); crm_log_xml_notice(msg, "BadEvent"); } } free(data_addr); return event; } static void event_free(stonith_event_t * event) { free(event->id); free(event->type); free(event->message); free(event->operation); free(event->origin); free(event->action); free(event->target); free(event->executioner); free(event->device); free(event->client_origin); free(event); } static void stonith_send_notification(gpointer data, gpointer user_data) { struct notify_blob_s *blob = user_data; stonith_notify_client_t *entry = data; stonith_event_t *st_event = NULL; const char *event = NULL; if (blob->xml == NULL) { crm_warn("Skipping callback - NULL message"); return; } event = crm_element_value(blob->xml, F_SUBTYPE); if (entry == NULL) { crm_warn("Skipping callback - NULL callback client"); return; } else if (entry->delete) { crm_trace("Skipping callback - marked for deletion"); return; } else if (entry->notify == NULL) { crm_warn("Skipping callback - NULL callback"); return; } else if (safe_str_neq(entry->event, event)) { crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } st_event = xml_to_event(blob->xml); crm_trace("Invoking callback for %p/%s event...", entry, event); entry->notify(blob->stonith, st_event); crm_trace("Callback invoked..."); event_free(st_event); } /*! * \internal * \brief Create and send an API request * * \param[in] stonith Stonith connection * \param[in] op API operation to request * \param[in] data Data to attach to request * \param[out] output_data If not NULL, will be set to reply if synchronous * \param[in] call_options Bitmask of stonith_call_options to use * \param[in] timeout Error if not completed within this many seconds * * \return pcmk_ok (for synchronous requests) or positive call ID * (for asynchronous requests) on success, -errno otherwise */ static int stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNode ** output_data, int call_options, int timeout) { int rc = 0; int reply_id = -1; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; stonith_private_t *native = NULL; CRM_ASSERT(stonith && stonith->st_private && op); native = stonith->st_private; if (output_data != NULL) { *output_data = NULL; } if ((stonith->state == stonith_disconnected) || (native->token == NULL)) { return -ENOTCONN; } /* Increment the call ID, which must be positive to avoid conflicting with * error codes. This shouldn't be a problem unless the client mucked with * it or the counter wrapped around. */ stonith->call_id++; if (stonith->call_id < 1) { stonith->call_id = 1; } op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options); if (op_msg == NULL) { return -EINVAL; } crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout); crm_trace("Sending %s message to fencer with timeout %ds", op, timeout); { enum crm_ipc_flags ipc_flags = crm_ipc_flags_none; if (call_options & st_opt_sync_call) { ipc_flags |= crm_ipc_client_response; } rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, 1000 * (timeout + 60), &op_reply); } free_xml(op_msg); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc); rc = -ECOMM; goto done; } crm_log_xml_trace(op_reply, "Reply"); if (!(call_options & st_opt_sync_call)) { crm_trace("Async call %d, returning", stonith->call_id); free_xml(op_reply); return stonith->call_id; } rc = pcmk_ok; crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id); if (reply_id == stonith->call_id) { crm_trace("Synchronous reply %d received", reply_id); if (crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) { rc = -ENOMSG; } if ((call_options & st_opt_discard_reply) || output_data == NULL) { crm_trace("Discarding reply"); } else { *output_data = op_reply; op_reply = NULL; /* Prevent subsequent free */ } } else if (reply_id <= 0) { crm_err("Received bad reply: No id set"); crm_log_xml_err(op_reply, "Bad reply"); free_xml(op_reply); rc = -ENOMSG; } else { crm_err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id); crm_log_xml_err(op_reply, "Old reply"); free_xml(op_reply); rc = -ENOMSG; } done: if (crm_ipc_connected(native->ipc) == FALSE) { crm_err("Fencer disconnected"); free(native->token); native->token = NULL; stonith->state = stonith_disconnected; } free_xml(op_reply); return rc; } /* Not used with mainloop */ bool stonith_dispatch(stonith_t * st) { gboolean stay_connected = TRUE; stonith_private_t *private = NULL; CRM_ASSERT(st != NULL); private = st->st_private; while (crm_ipc_ready(private->ipc)) { if (crm_ipc_read(private->ipc) > 0) { const char *msg = crm_ipc_buffer(private->ipc); stonith_dispatch_internal(msg, strlen(msg), st); } if (crm_ipc_connected(private->ipc) == FALSE) { crm_err("Connection closed"); stay_connected = FALSE; } } return stay_connected; } static int stonith_api_free(stonith_t * stonith) { int rc = pcmk_ok; crm_trace("Destroying %p", stonith); if (stonith->state != stonith_disconnected) { crm_trace("Disconnecting %p first", stonith); rc = stonith->cmds->disconnect(stonith); } if (stonith->state == stonith_disconnected) { stonith_private_t *private = stonith->st_private; crm_trace("Removing %d callbacks", g_hash_table_size(private->stonith_op_callback_table)); g_hash_table_destroy(private->stonith_op_callback_table); crm_trace("Destroying %d notification clients", g_list_length(private->notify_list)); g_list_free_full(private->notify_list, free); free(stonith->st_private); free(stonith->cmds); free(stonith); } else { crm_err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc); } return rc; } void stonith_api_delete(stonith_t * stonith) { crm_trace("Destroying %p", stonith); if(stonith) { stonith->cmds->free(stonith); } } static int stonith_api_validate(stonith_t *st, int call_options, const char *rsc_id, const char *namespace_s, const char *agent, stonith_key_value_t *params, int timeout, char **output, char **error_output) { /* Validation should be done directly via the agent, so we can get it from * stonith_admin when the cluster is not running, which is important for * higher-level tools. */ int rc = pcmk_ok; /* Use a dummy node name in case the agent requires a target. We assume the * actual target doesn't matter for validation purposes (if in practice, * that is incorrect, we will need to allow the caller to pass the target). */ const char *target = "node1"; GHashTable *params_table = crm_str_table_new(); // Convert parameter list to a hash table for (; params; params = params->next) { // Strip out Pacemaker-implemented parameters if (!crm_starts_with(params->key, "pcmk_") && strcmp(params->key, "provides") && strcmp(params->key, "stonith-timeout")) { g_hash_table_insert(params_table, strdup(params->key), strdup(params->value)); } } #if SUPPORT_CIBSECRETS rc = replace_secret_params(rsc_id, params_table); if (rc < 0) { crm_warn("Could not replace secret parameters for validation of %s: %s", agent, pcmk_strerror(rc)); } #endif if (output) { *output = NULL; } if (error_output) { *error_output = NULL; } switch (stonith_get_namespace(agent, namespace_s)) { case st_namespace_rhcs: rc = stonith__rhcs_validate(st, call_options, target, agent, params_table, timeout, output, error_output); break; #if HAVE_STONITH_STONITH_H case st_namespace_lha: rc = stonith__lha_validate(st, call_options, target, agent, params_table, timeout, output, error_output); break; #endif default: rc = -EINVAL; errno = EINVAL; crm_perror(LOG_ERR, "Agent %s not found or does not support validation", agent); break; } g_hash_table_destroy(params_table); return rc; } stonith_t * stonith_api_new(void) { stonith_t *new_stonith = NULL; stonith_private_t *private = NULL; new_stonith = calloc(1, sizeof(stonith_t)); if (new_stonith == NULL) { return NULL; } private = calloc(1, sizeof(stonith_private_t)); if (private == NULL) { free(new_stonith); return NULL; } new_stonith->st_private = private; private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); private->notify_list = NULL; private->notify_refcnt = 0; private->notify_deletes = FALSE; new_stonith->call_id = 1; new_stonith->state = stonith_disconnected; new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t)); if (new_stonith->cmds == NULL) { free(new_stonith->st_private); free(new_stonith); return NULL; } /* *INDENT-OFF* */ new_stonith->cmds->free = stonith_api_free; new_stonith->cmds->connect = stonith_api_signon; new_stonith->cmds->disconnect = stonith_api_signoff; new_stonith->cmds->list = stonith_api_list; new_stonith->cmds->monitor = stonith_api_monitor; new_stonith->cmds->status = stonith_api_status; new_stonith->cmds->fence = stonith_api_fence; new_stonith->cmds->confirm = stonith_api_confirm; new_stonith->cmds->history = stonith_api_history; new_stonith->cmds->list_agents = stonith_api_device_list; new_stonith->cmds->metadata = stonith_api_device_metadata; new_stonith->cmds->query = stonith_api_query; new_stonith->cmds->remove_device = stonith_api_remove_device; new_stonith->cmds->register_device = stonith_api_register_device; new_stonith->cmds->remove_level = stonith_api_remove_level; new_stonith->cmds->remove_level_full = stonith_api_remove_level_full; new_stonith->cmds->register_level = stonith_api_register_level; new_stonith->cmds->register_level_full = stonith_api_register_level_full; new_stonith->cmds->remove_callback = stonith_api_del_callback; new_stonith->cmds->register_callback = stonith_api_add_callback; new_stonith->cmds->remove_notification = stonith_api_del_notification; new_stonith->cmds->register_notification = stonith_api_add_notification; new_stonith->cmds->validate = stonith_api_validate; /* *INDENT-ON* */ return new_stonith; } /*! * \brief Make a blocking connection attempt to the fencer * * \param[in,out] st Fencer API object * \param[in] name Client name to use with fencer * \param[in] max_attempts Return error if this many attempts fail * * \return pcmk_ok on success, result of last attempt otherwise */ int stonith_api_connect_retry(stonith_t *st, const char *name, int max_attempts) { int rc = -EINVAL; // if max_attempts is not positive for (int attempt = 1; attempt <= max_attempts; attempt++) { rc = st->cmds->connect(st, name, NULL); if (rc == pcmk_ok) { return pcmk_ok; } else if (attempt < max_attempts) { crm_notice("Fencer connection attempt %d of %d failed (retrying in 2s): %s " CRM_XS " rc=%d", attempt, max_attempts, pcmk_strerror(rc), rc); sleep(2); } } crm_notice("Could not connect to fencer: %s " CRM_XS " rc=%d", pcmk_strerror(rc), rc); return rc; } stonith_key_value_t * stonith_key_value_add(stonith_key_value_t * head, const char *key, const char *value) { stonith_key_value_t *p, *end; p = calloc(1, sizeof(stonith_key_value_t)); if (key) { p->key = strdup(key); } if (value) { p->value = strdup(value); } end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } void stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values) { stonith_key_value_t *p; while (head) { p = head->next; if (keys) { free(head->key); } if (values) { free(head->value); } free(head); head = p; } } #define api_log_open() openlog("stonith-api", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON) #define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __FUNCTION__, args) int stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off) { int rc = pcmk_ok; stonith_t *st = stonith_api_new(); const char *action = off? "off" : "reboot"; api_log_open(); if (st == NULL) { api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s", action, nodeid, uname); return -EPROTO; } rc = st->cmds->connect(st, "stonith-api", NULL); if (rc != pcmk_ok) { api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); } else { char *name = NULL; enum stonith_call_options opts = st_opt_sync_call | st_opt_allow_suicide; if (uname != NULL) { name = strdup(uname); } else if (nodeid > 0) { opts |= st_opt_cs_nodeid; name = crm_itoa(nodeid); } rc = st->cmds->fence(st, opts, name, action, timeout, 0); free(name); if (rc != pcmk_ok) { api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); } else { api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action); } } stonith_api_delete(st); return rc; } time_t stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress) { int rc = pcmk_ok; time_t when = 0; stonith_t *st = stonith_api_new(); stonith_history_t *history = NULL, *hp = NULL; if (st == NULL) { api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: " "API initialization failed", nodeid, uname); return when; } rc = st->cmds->connect(st, "stonith-api", NULL); if (rc != pcmk_ok) { api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc); } else { int entries = 0; int progress = 0; int completed = 0; char *name = NULL; enum stonith_call_options opts = st_opt_sync_call; if (uname != NULL) { name = strdup(uname); } else if (nodeid > 0) { opts |= st_opt_cs_nodeid; name = crm_itoa(nodeid); } rc = st->cmds->history(st, opts, name, &history, 120); free(name); for (hp = history; hp; hp = hp->next) { entries++; if (in_progress) { progress++; if (hp->state != st_done && hp->state != st_failed) { when = time(NULL); } } else if (hp->state == st_done) { completed++; if (hp->completed > when) { when = hp->completed; } } } stonith_history_free(history); if(rc == pcmk_ok) { api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed); } else { api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc); } } stonith_api_delete(st); if(when) { api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when); } return when; } bool stonith_agent_exists(const char *agent, int timeout) { stonith_t *st = NULL; stonith_key_value_t *devices = NULL; stonith_key_value_t *dIter = NULL; bool rc = FALSE; if (agent == NULL) { return rc; } st = stonith_api_new(); if (st == NULL) { crm_err("Could not list fence agents: API memory allocation failed"); return FALSE; } st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout == 0 ? 120 : timeout); for (dIter = devices; dIter != NULL; dIter = dIter->next) { if (crm_str_eq(dIter->value, agent, TRUE)) { rc = TRUE; break; } } stonith_key_value_freeall(devices, 1, 1); stonith_api_delete(st); return rc; } const char * stonith_action_str(const char *action) { if (action == NULL) { return "fencing"; } else if (!strcmp(action, "on")) { return "unfencing"; } else if (!strcmp(action, "off")) { return "turning off"; } else { return action; } } /*! * \internal * \brief Parse a target name from one line of a target list string * * \param[in] line One line of a target list string * \parma[in] len String length of line * \param[in,out] output List to add newly allocated target name to */ static void parse_list_line(const char *line, int len, GList **output) { size_t i = 0; size_t entry_start = 0; /* Skip complaints about additional parameters device doesn't understand * * @TODO Document or eliminate the implied restriction of target names */ if (strstr(line, "invalid") || strstr(line, "variable")) { crm_debug("Skipping list output line: %s", line); return; } // Process line content, character by character for (i = 0; i <= len; i++) { if (isspace(line[i]) || (line[i] == ',') || (line[i] == ';') || (line[i] == '\0')) { // We've found a separator (i.e. the end of an entry) int rc = 0; char *entry = NULL; if (i == entry_start) { // Skip leading and sequential separators entry_start = i + 1; continue; } entry = calloc(i - entry_start + 1, sizeof(char)); CRM_ASSERT(entry != NULL); /* Read entry, stopping at first separator * * @TODO Document or eliminate these character restrictions */ rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry); if (rc != 1) { crm_warn("Could not parse list output entry: %s " CRM_XS " entry_start=%d position=%d", line + entry_start, entry_start, i); free(entry); } else if (safe_str_eq(entry, "on") || safe_str_eq(entry, "off")) { /* Some agents print the target status in the list output, * though none are known now (the separate list-status command * is used for this, but it can also print "UNKNOWN"). To handle * this possibility, skip such entries. * * @TODO Document or eliminate the implied restriction of target * names. */ free(entry); } else { // We have a valid entry *output = g_list_append(*output, entry); } entry_start = i + 1; } } } /*! * \internal * \brief Parse a list of targets from a string * * \param[in] list_output Target list as a string * * \return List of target names * \note The target list string format is flexible, to allow for user-specified * lists such pcmk_host_list and the output of an agent's list action * (whether direct or via the API, which escapes newlines). There may be * multiple lines, separated by either a newline or an escaped newline * (backslash n). Each line may have one or more target names, separated * by any combination of whitespace, commas, and semi-colons. Lines * containing "invalid" or "variable" will be ignored entirely. Target * names "on" or "off" (case-insensitive) will be ignored. Target names * may contain only alphanumeric characters, underbars (_), dashes (-), * and dots (.) (if any other character occurs in the name, it and all * subsequent characters in the name will be ignored). * \note The caller is responsible for freeing the result with * g_list_free_full(result, free). */ GList * stonith__parse_targets(const char *target_spec) { GList *targets = NULL; if (target_spec != NULL) { size_t out_len = strlen(target_spec); size_t line_start = 0; // Starting index of line being processed for (size_t i = 0; i <= out_len; ++i) { if ((target_spec[i] == '\n') || (target_spec[i] == '\0') || ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) { // We've reached the end of one line of output int len = i - line_start; if (len > 0) { char *line = strndup(target_spec + line_start, len); line[len] = '\0'; // Because it might be a newline parse_list_line(line, len, &targets); free(line); } if (target_spec[i] == '\\') { ++i; // backslash-n takes up two positions } line_start = i + 1; } } } return targets; } + +/*! + * \internal + * \brief Determine if a later stonith event succeeded. + */ +gboolean +stonith__later_succeeded(stonith_history_t *event, stonith_history_t *top_history) +{ + gboolean ret = FALSE; + + for (stonith_history_t *prev_hp = top_history; prev_hp; prev_hp = prev_hp->next) { + if (prev_hp == event) { + break; + } + + if ((prev_hp->state == st_done) && + safe_str_eq(event->target, prev_hp->target) && + safe_str_eq(event->action, prev_hp->action) && + safe_str_eq(event->delegate, prev_hp->delegate) && + (event->completed < prev_hp->completed)) { + ret = TRUE; + break; + } + } + return ret; +} diff --git a/lib/fencing/st_output.c b/lib/fencing/st_output.c index 2cf62e16e8..b2bb20f2f6 100644 --- a/lib/fencing/st_output.c +++ b/lib/fencing/st_output.c @@ -1,284 +1,291 @@ /* * Copyright 2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include static char * time_t_string(time_t when) { crm_time_t *crm_when = crm_time_new(NULL); char *buf = NULL; crm_time_set_timet(crm_when, &when); buf = crm_time_as_string(crm_when, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); crm_time_free(crm_when); return buf; } static int last_fenced_html(pcmk__output_t *out, va_list args) { const char *target = va_arg(args, const char *); time_t when = va_arg(args, time_t); if (when) { char *buf = crm_strdup_printf("Node %s last fenced at: %s", target, ctime(&when)); pcmk__output_create_html_node(out, "div", NULL, NULL, buf); free(buf); } return 0; } static int last_fenced_text(pcmk__output_t *out, va_list args) { const char *target = va_arg(args, const char *); time_t when = va_arg(args, time_t); if (when) { pcmk__indented_printf(out, "Node %s last fenced at: %s", target, ctime(&when)); } else { pcmk__indented_printf(out, "Node %s has never been fenced\n", target); } return 0; } static int last_fenced_xml(pcmk__output_t *out, va_list args) { const char *target = va_arg(args, const char *); time_t when = va_arg(args, time_t); if (when) { xmlNodePtr node = pcmk__output_create_xml_node(out, "last-fenced"); char *buf = time_t_string(when); xmlSetProp(node, (pcmkXmlStr) "target", (pcmkXmlStr) target); xmlSetProp(node, (pcmkXmlStr) "when", (pcmkXmlStr) buf); free(buf); } return 0; } static int stonith_event_html(pcmk__output_t *out, va_list args) { stonith_history_t *event = va_arg(args, stonith_history_t *); int full_history = va_arg(args, int); + gboolean later_succeeded = va_arg(args, gboolean); + char *buf = NULL; switch(event->state) { case st_done: buf = crm_strdup_printf("%s of %s successful: delegate=%s, client=%s, origin=%s, %s='%s'", stonith_action_str(event->action), event->target, event->delegate ? event->delegate : "", event->client, event->origin, full_history ? "completed" : "last-successful", time_t_string(event->completed)); out->list_item(out, "successful-stonith-event", buf); free(buf); break; case st_failed: - buf = crm_strdup_printf("%s of %s failed : delegate=%s, client=%s, origin=%s, %s='%s'", + buf = crm_strdup_printf("%s of %s failed : delegate=%s, client=%s, origin=%s, %s='%s' %s", stonith_action_str(event->action), event->target, event->delegate ? event->delegate : "", event->client, event->origin, full_history ? "completed" : "last-failed", - time_t_string(event->completed)); + time_t_string(event->completed), + later_succeeded ? "(a later attempt succeeded)" : ""); out->list_item(out, "failed-stonith-event", buf); free(buf); break; default: buf = crm_strdup_printf("%s of %s pending: client=%s, origin=%s", stonith_action_str(event->action), event->target, event->client, event->origin); out->list_item(out, "pending-stonith-event", buf); free(buf); break; } return 0; } static int stonith_event_text(pcmk__output_t *out, va_list args) { stonith_history_t *event = va_arg(args, stonith_history_t *); int full_history = va_arg(args, int); + gboolean later_succeeded = va_arg(args, gboolean); + char *buf = time_t_string(event->completed); switch (event->state) { case st_failed: - pcmk__indented_printf(out, "%s of %s failed: delegate=%s, client=%s, origin=%s, %s='%s'\n", + pcmk__indented_printf(out, "%s of %s failed: delegate=%s, client=%s, origin=%s, %s='%s' %s\n", stonith_action_str(event->action), event->target, event->delegate ? event->delegate : "", event->client, event->origin, - full_history ? "completed" : "last-failed", buf); + full_history ? "completed" : "last-failed", buf, + later_succeeded ? "(a later attempt succeeded)" : ""); break; case st_done: pcmk__indented_printf(out, "%s of %s successful: delegate=%s, client=%s, origin=%s, %s='%s'\n", stonith_action_str(event->action), event->target, event->delegate ? event->delegate : "", event->client, event->origin, full_history ? "completed" : "last-successful", buf); break; default: pcmk__indented_printf(out, "%s of %s pending: client=%s, origin=%s\n", stonith_action_str(event->action), event->target, event->client, event->origin); break; } free(buf); return 0; } static int stonith_event_xml(pcmk__output_t *out, va_list args) { xmlNodePtr node = pcmk__output_create_xml_node(out, "fence_event"); stonith_history_t *event = va_arg(args, stonith_history_t *); + char *buf = NULL; switch (event->state) { case st_failed: xmlSetProp(node, (pcmkXmlStr) "status", (pcmkXmlStr) "failed"); break; case st_done: xmlSetProp(node, (pcmkXmlStr) "status", (pcmkXmlStr) "success"); break; default: { char *state = crm_itoa(event->state); xmlSetProp(node, (pcmkXmlStr) "status", (pcmkXmlStr) "pending"); xmlSetProp(node, (pcmkXmlStr) "extended-status", (pcmkXmlStr) state); free(state); break; } } if (event->delegate != NULL) { xmlSetProp(node, (pcmkXmlStr) "delegate", (pcmkXmlStr) event->delegate); } xmlSetProp(node, (pcmkXmlStr) "action", (pcmkXmlStr) event->action); xmlSetProp(node, (pcmkXmlStr) "target", (pcmkXmlStr) event->target); xmlSetProp(node, (pcmkXmlStr) "client", (pcmkXmlStr) event->client); xmlSetProp(node, (pcmkXmlStr) "origin", (pcmkXmlStr) event->origin); if (event->state == st_failed || event->state == st_done) { buf = time_t_string(event->completed); xmlSetProp(node, (pcmkXmlStr) "completed", (pcmkXmlStr) buf); free(buf); } return 0; } static int validate_agent_html(pcmk__output_t *out, va_list args) { const char *agent = va_arg(args, const char *); const char *device = va_arg(args, const char *); const char *output = va_arg(args, const char *); const char *error_output = va_arg(args, const char *); int rc = va_arg(args, int); if (device) { char *buf = crm_strdup_printf("Validation of %s on %s %s", agent, device, rc ? "failed" : "succeeded"); pcmk__output_create_html_node(out, "div", NULL, NULL, buf); free(buf); } else { char *buf = crm_strdup_printf("Validation of %s %s", agent, rc ? "failed" : "succeeded"); pcmk__output_create_html_node(out, "div", NULL, NULL, buf); free(buf); } out->subprocess_output(out, rc, output, error_output); return rc; } static int validate_agent_text(pcmk__output_t *out, va_list args) { const char *agent = va_arg(args, const char *); const char *device = va_arg(args, const char *); const char *output = va_arg(args, const char *); const char *error_output = va_arg(args, const char *); int rc = va_arg(args, int); if (device) { pcmk__indented_printf(out, "Validation of %s on %s %s\n", agent, device, rc ? "failed" : "succeeded"); } else { pcmk__indented_printf(out, "Validation of %s %s\n", agent, rc ? "failed" : "succeeded"); } if (output) { puts(output); } if (error_output) { puts(error_output); } return rc; } static int validate_agent_xml(pcmk__output_t *out, va_list args) { xmlNodePtr node = pcmk__output_create_xml_node(out, "validate"); const char *agent = va_arg(args, const char *); const char *device = va_arg(args, const char *); const char *output = va_arg(args, const char *); const char *error_output = va_arg(args, const char *); int rc = va_arg(args, int); xmlSetProp(node, (pcmkXmlStr) "agent", (pcmkXmlStr) agent); if (device != NULL) { xmlSetProp(node, (pcmkXmlStr) "device", (pcmkXmlStr) device); } xmlSetProp(node, (pcmkXmlStr) "valid", (pcmkXmlStr) (rc ? "false" : "true")); pcmk__output_xml_push_parent(out, node); out->subprocess_output(out, rc, output, error_output); pcmk__output_xml_pop_parent(out); return rc; } static pcmk__message_entry_t fmt_functions[] = { { "last-fenced", "html", last_fenced_html }, { "last-fenced", "text", last_fenced_text }, { "last-fenced", "xml", last_fenced_xml }, { "stonith-event", "html", stonith_event_html }, { "stonith-event", "text", stonith_event_text }, { "stonith-event", "xml", stonith_event_xml }, { "validate", "html", validate_agent_html }, { "validate", "text", validate_agent_text }, { "validate", "xml", validate_agent_xml }, { NULL, NULL, NULL } }; void stonith__register_messages(pcmk__output_t *out) { pcmk__register_messages(out, fmt_functions); } diff --git a/tools/crm_mon_print.c b/tools/crm_mon_print.c index 192f034ef2..f6e73853b8 100644 --- a/tools/crm_mon_print.c +++ b/tools/crm_mon_print.c @@ -1,2625 +1,2603 @@ /* * Copyright 2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include "crm_mon.h" static void print_nvpair(mon_state_t *state, const char *name, const char *value, const char *units, time_t epoch_time); static void print_node_start(mon_state_t *state, node_t *node, unsigned int mon_ops); static void print_node_end(mon_state_t *state); static void print_resources_heading(mon_state_t *state, unsigned int mon_ops); static void print_resources_closing(mon_state_t *state, gboolean printed_heading, unsigned int mon_ops); static void print_resources(mon_state_t *state, pe_working_set_t *data_set, int print_opts, unsigned int mon_ops); static void print_rsc_history_start(mon_state_t *state, pe_working_set_t *data_set, node_t *node, resource_t *rsc, const char *rsc_id, gboolean all); static void print_rsc_history_end(mon_state_t *state); static void print_op_history(mon_state_t *state, pe_working_set_t *data_set, node_t *node, xmlNode *xml_op, const char *task, const char *interval_ms_s, int rc, unsigned int mon_ops); static void print_rsc_history(mon_state_t *state, pe_working_set_t *data_set, node_t *node, xmlNode *rsc_entry, gboolean operations, unsigned int mon_ops); static void print_node_history(mon_state_t *state, pe_working_set_t *data_set, xmlNode *node_state, gboolean operations, unsigned int mon_ops); static gboolean print_attr_msg(mon_state_t *state, node_t * node, GListPtr rsc_list, const char *attrname, const char *attrvalue); static void print_node_attribute(gpointer name, gpointer user_data); static void print_node_summary(mon_state_t *state, pe_working_set_t * data_set, gboolean operations, unsigned int mon_ops); static void print_ticket(gpointer name, gpointer value, gpointer user_data); static void print_cluster_tickets(mon_state_t *state, pe_working_set_t * data_set); static void print_ban(mon_state_t *state, pe_node_t *node, pe__location_t *location, unsigned int mon_ops); static void print_neg_locations(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops, const char *prefix); static void print_node_attributes(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops); static void print_cluster_summary_header(mon_state_t *state); static void print_cluster_summary_footer(mon_state_t *state); static void print_cluster_times(mon_state_t *state, pe_working_set_t *data_set); static void print_cluster_stack(mon_state_t *state, const char *stack_s); static void print_cluster_dc(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops); static void print_cluster_counts(mon_state_t *state, pe_working_set_t *data_set, const char *stack_s); static void print_cluster_options(mon_state_t *state, pe_working_set_t *data_set); static void print_cluster_summary(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops, unsigned int show); static void print_failed_action(mon_state_t *state, xmlNode *xml_op); static void print_failed_actions(mon_state_t *state, pe_working_set_t *data_set); static void print_stonith_action(mon_state_t *state, stonith_history_t *event, unsigned int mon_ops, stonith_history_t *top_history); static void print_failed_stonith_actions(mon_state_t *state, stonith_history_t *history, unsigned int mon_ops); static void print_stonith_pending(mon_state_t *state, stonith_history_t *history, unsigned int mon_ops); static void print_stonith_history(mon_state_t *state, stonith_history_t *history, unsigned int mon_ops); /*! * \internal * \brief Print a [name]=[value][units] pair, optionally using time string * * \param[in] stream File stream to display output to * \param[in] name Name to display * \param[in] value Value to display (or NULL to convert time instead) * \param[in] units Units to display (or NULL for no units) * \param[in] epoch_time Epoch time to convert if value is NULL */ static void print_nvpair(mon_state_t *state, const char *name, const char *value, const char *units, time_t epoch_time) { /* print name= */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " %s=", name); break; case mon_output_html: case mon_output_cgi: case mon_output_xml: fprintf(state->stream, " %s=", name); break; default: break; } /* If we have a value (and optionally units), print it */ if (value) { switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "%s%s", value, (units? units : "")); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "%s%s", value, (units? units : "")); break; case mon_output_xml: fprintf(state->stream, "\"%s%s\"", value, (units? units : "")); break; default: break; } /* Otherwise print user-friendly time string */ } else { static char empty_str[] = ""; char *c, *date_str = asctime(localtime(&epoch_time)); for (c = (date_str != NULL) ? date_str : empty_str; *c != '\0'; ++c) { if (*c == '\n') { *c = '\0'; break; } } switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "'%s'", date_str); break; case mon_output_html: case mon_output_cgi: case mon_output_xml: fprintf(state->stream, "\"%s\"", date_str); break; default: break; } } } /*! * \internal * \brief Print whatever is needed to start a node section * * \param[in] stream File stream to display output to * \param[in] node Node to print */ static void print_node_start(mon_state_t *state, node_t *node, unsigned int mon_ops) { char *node_name; switch (state->output_format) { case mon_output_plain: case mon_output_console: node_name = get_node_display_name(node, mon_ops); print_as(state->output_format, "* Node %s:\n", node_name); free(node_name); break; case mon_output_html: case mon_output_cgi: node_name = get_node_display_name(node, mon_ops); fprintf(state->stream, "

Node: %s

\n
    \n", node_name); free(node_name); break; case mon_output_xml: fprintf(state->stream, " \n", node->details->uname); break; default: break; } } /*! * \internal * \brief Print whatever is needed to end a node section * * \param[in] stream File stream to display output to */ static void print_node_end(mon_state_t *state) { switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
\n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print resources section heading appropriate to options * * \param[in] stream File stream to display output to */ static void print_resources_heading(mon_state_t *state, unsigned int mon_ops) { const char *heading; if (is_set(mon_ops, mon_op_group_by_node)) { /* Active resources have already been printed by node */ heading = is_set(mon_ops, mon_op_inactive_resources) ? "Inactive resources" : NULL; } else if (is_set(mon_ops, mon_op_inactive_resources)) { heading = "Full list of resources"; } else { heading = "Active resources"; } /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\n%s:\n\n", heading); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
\n

%s

\n", heading); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print whatever resource section closing is appropriate * * \param[in] stream File stream to display output to */ static void print_resources_closing(mon_state_t *state, gboolean printed_heading, unsigned int mon_ops) { const char *heading; /* What type of resources we did or did not display */ if (is_set(mon_ops, mon_op_group_by_node)) { heading = "inactive "; } else if (is_set(mon_ops, mon_op_inactive_resources)) { heading = ""; } else { heading = "active "; } switch (state->output_format) { case mon_output_plain: case mon_output_console: if (!printed_heading) { print_as(state->output_format, "\nNo %sresources\n\n", heading); } break; case mon_output_html: case mon_output_cgi: if (!printed_heading) { fprintf(state->stream, "
\n

No %sresources

\n", heading); } break; case mon_output_xml: fprintf(state->stream, " %s\n", (printed_heading? "" : "")); break; default: break; } } /*! * \internal * \brief Print whatever resource section(s) are appropriate * * \param[in] stream File stream to display output to * \param[in] data_set Cluster state to display * \param[in] print_opts Bitmask of pe_print_options */ static void print_resources(mon_state_t *state, pe_working_set_t *data_set, int print_opts, unsigned int mon_ops) { GListPtr rsc_iter; const char *prefix = NULL; gboolean printed_heading = FALSE; gboolean brief_output = is_set(mon_ops, mon_op_print_brief); /* If we already showed active resources by node, and * we're not showing inactive resources, we have nothing to do */ if (is_set(mon_ops, mon_op_group_by_node) && is_not_set(mon_ops, mon_op_inactive_resources)) { return; } /* XML uses an indent, and ignores brief option for resources */ if (state->output_format == mon_output_xml) { prefix = " "; brief_output = FALSE; } /* If we haven't already printed resources grouped by node, * and brief output was requested, print resource summary */ if (brief_output && is_not_set(mon_ops, mon_op_group_by_node)) { print_resources_heading(state, mon_ops); printed_heading = TRUE; print_rscs_brief(data_set->resources, NULL, print_opts, state->stream, is_set(mon_ops, mon_op_inactive_resources)); } /* For each resource, display it if appropriate */ for (rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) { resource_t *rsc = (resource_t *) rsc_iter->data; /* Complex resources may have some sub-resources active and some inactive */ gboolean is_active = rsc->fns->active(rsc, TRUE); gboolean partially_active = rsc->fns->active(rsc, FALSE); /* Skip inactive orphans (deleted but still in CIB) */ if (is_set(rsc->flags, pe_rsc_orphan) && !is_active) { continue; /* Skip active resources if we already displayed them by node */ } else if (is_set(mon_ops, mon_op_group_by_node)) { if (is_active) { continue; } /* Skip primitives already counted in a brief summary */ } else if (brief_output && (rsc->variant == pe_native)) { continue; /* Skip resources that aren't at least partially active, * unless we're displaying inactive resources */ } else if (!partially_active && is_not_set(mon_ops, mon_op_inactive_resources)) { continue; } /* Print this resource */ if (printed_heading == FALSE) { print_resources_heading(state, mon_ops); printed_heading = TRUE; } rsc->fns->print(rsc, prefix, print_opts, state->stream); } print_resources_closing(state, printed_heading, mon_ops); } /*! * \internal * \brief Print heading for resource history * * \param[in] stream File stream to display output to * \param[in] data_set Current state of CIB * \param[in] node Node that ran this resource * \param[in] rsc Resource to print * \param[in] rsc_id ID of resource to print * \param[in] all Whether to print every resource or just failed ones */ static void print_rsc_history_start(mon_state_t *state, pe_working_set_t *data_set, node_t *node, resource_t *rsc, const char *rsc_id, gboolean all) { time_t last_failure = 0; int failcount = rsc? pe_get_failcount(node, rsc, &last_failure, pe_fc_default, NULL, data_set) : 0; if (!all && !failcount && (last_failure <= 0)) { return; } /* Print resource ID */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " %s:", rsc_id); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
  • %s:", rsc_id); break; case mon_output_xml: fprintf(state->stream, " output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " orphan"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " orphan"); break; case mon_output_xml: fprintf(state->stream, " orphan=\"true\""); break; default: break; } /* If resource is not an orphan, print some details */ } else if (all || failcount || (last_failure > 0)) { /* Print migration threshold */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " migration-threshold=%d", rsc->migration_threshold); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " migration-threshold=%d", rsc->migration_threshold); break; case mon_output_xml: fprintf(state->stream, " orphan=\"false\" migration-threshold=\"%d\"", rsc->migration_threshold); break; default: break; } /* Print fail count if any */ if (failcount > 0) { switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " " CRM_FAIL_COUNT_PREFIX "=%d", failcount); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " " CRM_FAIL_COUNT_PREFIX "=%d", failcount); break; case mon_output_xml: fprintf(state->stream, " " CRM_FAIL_COUNT_PREFIX "=\"%d\"", failcount); break; default: break; } } /* Print last failure time if any */ if (last_failure > 0) { print_nvpair(state, CRM_LAST_FAILURE_PREFIX, NULL, NULL, last_failure); } } /* End the heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "\n
      \n"); break; case mon_output_xml: fprintf(state->stream, ">\n"); break; default: break; } } /*! * \internal * \brief Print closing for resource history * * \param[in] stream File stream to display output to */ static void print_rsc_history_end(mon_state_t *state) { switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n
    • \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print operation history * * \param[in] stream File stream to display output to * \param[in] data_set Current state of CIB * \param[in] node Node this operation is for * \param[in] xml_op Root of XML tree describing this operation * \param[in] task Task parsed from this operation's XML * \param[in] interval_ms_s Interval parsed from this operation's XML * \param[in] rc Return code parsed from this operation's XML */ static void print_op_history(mon_state_t *state, pe_working_set_t *data_set, node_t *node, xmlNode *xml_op, const char *task, const char *interval_ms_s, int rc, unsigned int mon_ops) { const char *value = NULL; const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID); /* Begin the operation description */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " + (%s) %s:", call, task); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
  • (%s) %s:", call, task); break; case mon_output_xml: fprintf(state->stream, " 0) { print_nvpair(state, attr, NULL, NULL, int_value); } } attr = XML_RSC_OP_LAST_RUN; value = crm_element_value(xml_op, attr); if (value) { int_value = crm_parse_int(value, NULL); if (int_value > 0) { print_nvpair(state, attr, NULL, NULL, int_value); } } attr = XML_RSC_OP_T_EXEC; value = crm_element_value(xml_op, attr); if (value) { print_nvpair(state, attr, value, "ms", 0); } attr = XML_RSC_OP_T_QUEUE; value = crm_element_value(xml_op, attr); if (value) { print_nvpair(state, attr, value, "ms", 0); } } /* End the operation description */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, " rc=%d (%s)\n", rc, services_ocf_exitcode_str(rc)); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " rc=%d (%s)
    • \n", rc, services_ocf_exitcode_str(rc)); break; case mon_output_xml: fprintf(state->stream, " rc=\"%d\" rc_text=\"%s\" />\n", rc, services_ocf_exitcode_str(rc)); break; default: break; } } /*! * \internal * \brief Print resource operation/failure history * * \param[in] stream File stream to display output to * \param[in] data_set Current state of CIB * \param[in] node Node that ran this resource * \param[in] rsc_entry Root of XML tree describing resource status * \param[in] operations Whether to print operations or just failcounts */ static void print_rsc_history(mon_state_t *state, pe_working_set_t *data_set, node_t *node, xmlNode *rsc_entry, gboolean operations, unsigned int mon_ops) { GListPtr gIter = NULL; GListPtr op_list = NULL; gboolean printed = FALSE; const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID); resource_t *rsc = pe_find_resource(data_set->resources, rsc_id); xmlNode *rsc_op = NULL; /* If we're not showing operations, just print the resource failure summary */ if (operations == FALSE) { print_rsc_history_start(state, data_set, node, rsc, rsc_id, FALSE); print_rsc_history_end(state); return; } /* Create a list of this resource's operations */ for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) { if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) { op_list = g_list_append(op_list, rsc_op); } } op_list = g_list_sort(op_list, sort_op_by_callid); /* Print each operation */ for (gIter = op_list; gIter != NULL; gIter = gIter->next) { xmlNode *xml_op = (xmlNode *) gIter->data; const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); const char *interval_ms_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS); const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC); int rc = crm_parse_int(op_rc, "0"); /* Display 0-interval monitors as "probe" */ if (safe_str_eq(task, CRMD_ACTION_STATUS) && ((interval_ms_s == NULL) || safe_str_eq(interval_ms_s, "0"))) { task = "probe"; } /* Ignore notifies and some probes */ if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || (safe_str_eq(task, "probe") && (rc == 7))) { continue; } /* If this is the first printed operation, print heading for resource */ if (printed == FALSE) { printed = TRUE; print_rsc_history_start(state, data_set, node, rsc, rsc_id, TRUE); } /* Print the operation */ print_op_history(state, data_set, node, xml_op, task, interval_ms_s, rc, mon_ops); } /* Free the list we created (no need to free the individual items) */ g_list_free(op_list); /* If we printed anything, close the resource */ if (printed) { print_rsc_history_end(state); } } /*! * \internal * \brief Print node operation/failure history * * \param[in] stream File stream to display output to * \param[in] data_set Current state of CIB * \param[in] node_state Root of XML tree describing node status * \param[in] operations Whether to print operations or just failcounts */ static void print_node_history(mon_state_t *state, pe_working_set_t *data_set, xmlNode *node_state, gboolean operations, unsigned int mon_ops) { node_t *node = pe_find_node_id(data_set->nodes, ID(node_state)); xmlNode *lrm_rsc = NULL; xmlNode *rsc_entry = NULL; if (node && node->details && node->details->online) { print_node_start(state, node, mon_ops); lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE); lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE); /* Print history of each of the node's resources */ for (rsc_entry = __xml_first_child_element(lrm_rsc); rsc_entry != NULL; rsc_entry = __xml_next_element(rsc_entry)) { if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) { print_rsc_history(state, data_set, node, rsc_entry, operations, mon_ops); } } print_node_end(state); } } /*! * \internal * \brief Print extended information about an attribute if appropriate * * \param[in] data_set Working set of CIB state * * \return TRUE if extended information was printed, FALSE otherwise * \note Currently, extended information is only supported for ping/pingd * resources, for which a message will be printed if connectivity is lost * or degraded. */ static gboolean print_attr_msg(mon_state_t *state, node_t * node, GListPtr rsc_list, const char *attrname, const char *attrvalue) { GListPtr gIter = NULL; for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; const char *type = g_hash_table_lookup(rsc->meta, "type"); if (rsc->children != NULL) { if (print_attr_msg(state, node, rsc->children, attrname, attrvalue)) { return TRUE; } } if (safe_str_eq(type, "ping") || safe_str_eq(type, "pingd")) { const char *name = g_hash_table_lookup(rsc->parameters, "name"); if (name == NULL) { name = "pingd"; } /* To identify the resource with the attribute name. */ if (safe_str_eq(name, attrname)) { int host_list_num = 0; int expected_score = 0; int value = crm_parse_int(attrvalue, "0"); const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list"); const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier"); if(hosts) { char **host_list = g_strsplit(hosts, " ", 0); host_list_num = g_strv_length(host_list); g_strfreev(host_list); } /* pingd multiplier is the same as the default value. */ expected_score = host_list_num * crm_parse_int(multiplier, "1"); switch (state->output_format) { case mon_output_plain: case mon_output_console: if (value <= 0) { print_as(state->output_format, "\t: Connectivity is lost"); } else if (value < expected_score) { print_as(state->output_format, "\t: Connectivity is degraded (Expected=%d)", expected_score); } break; case mon_output_html: case mon_output_cgi: if (value <= 0) { fprintf(state->stream, " (connectivity is lost)"); } else if (value < expected_score) { fprintf(state->stream, " (connectivity is degraded -- expected %d)", expected_score); } break; case mon_output_xml: fprintf(state->stream, " expected=\"%d\"", expected_score); break; default: break; } return TRUE; } } } return FALSE; } /* structure for passing multiple user data to g_list_foreach() */ struct mon_attr_data { mon_state_t *state; node_t *node; }; static void print_node_attribute(gpointer name, gpointer user_data) { const char *value = NULL; struct mon_attr_data *data = (struct mon_attr_data *) user_data; value = pe_node_attribute_raw(data->node, name); /* Print attribute name and value */ switch (data->state->output_format) { case mon_output_plain: case mon_output_console: print_as(data->state->output_format, " + %-32s\t: %-10s", (char *)name, value); break; case mon_output_html: case mon_output_cgi: fprintf(data->state->stream, "
  • %s: %s", (char *)name, value); break; case mon_output_xml: fprintf(data->state->stream, " state, data->node, data->node->details->running_rsc, name, value); /* Close out the attribute */ switch (data->state->output_format) { case mon_output_plain: case mon_output_console: print_as(data->state->output_format, "\n"); break; case mon_output_html: case mon_output_cgi: fprintf(data->state->stream, "
    • \n"); break; case mon_output_xml: fprintf(data->state->stream, " />\n"); break; default: break; } } static void print_node_summary(mon_state_t *state, pe_working_set_t * data_set, gboolean operations, unsigned int mon_ops) { xmlNode *node_state = NULL; xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input); /* Print heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: if (operations) { print_as(state->output_format, "\nOperations:\n"); } else { print_as(state->output_format, "\nMigration Summary:\n"); } break; case mon_output_html: case mon_output_cgi: if (operations) { fprintf(state->stream, "
    \n

    Operations

    \n"); } else { fprintf(state->stream, "
    \n

    Migration Summary

    \n"); } break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } /* Print each node in the CIB status */ for (node_state = __xml_first_child_element(cib_status); node_state != NULL; node_state = __xml_next_element(node_state)) { if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) { print_node_history(state, data_set, node_state, operations, mon_ops); } } /* Close section */ switch (state->output_format) { case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } static void print_ticket(gpointer name, gpointer value, gpointer user_data) { mon_state_t *data = (mon_state_t *) user_data; ticket_t *ticket = (ticket_t *) value; switch (data->output_format) { case mon_output_plain: case mon_output_console: print_as(data->output_format, "* %s:\t%s%s", ticket->id, (ticket->granted? "granted" : "revoked"), (ticket->standby? " [standby]" : "")); break; case mon_output_html: case mon_output_cgi: fprintf(data->stream, "
  • %s: %s%s", ticket->id, (ticket->granted? "granted" : "revoked"), (ticket->standby? " [standby]" : "")); break; case mon_output_xml: fprintf(data->stream, " id, (ticket->granted? "granted" : "revoked"), (ticket->standby? "true" : "false")); break; default: break; } if (ticket->last_granted > -1) { print_nvpair(data, "last-granted", NULL, NULL, ticket->last_granted); } switch (data->output_format) { case mon_output_plain: case mon_output_console: print_as(data->output_format, "\n"); break; case mon_output_html: case mon_output_cgi: fprintf(data->stream, "
    • \n"); break; case mon_output_xml: fprintf(data->stream, " />\n"); break; default: break; } } static void print_cluster_tickets(mon_state_t *state, pe_working_set_t * data_set) { /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nTickets:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Tickets

    \n
      \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } /* Print each ticket */ g_hash_table_foreach(data_set->tickets, print_ticket, state); /* Close section */ switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print a negative location constraint * * \param[in] stream File stream to display output to * \param[in] node Node affected by constraint * \param[in] location Constraint to print */ static void print_ban(mon_state_t *state, pe_node_t *node, pe__location_t *location, unsigned int mon_ops) { char *node_name = NULL; switch (state->output_format) { case mon_output_plain: case mon_output_console: node_name = get_node_display_name(node, mon_ops); print_as(state->output_format, " %s\tprevents %s from running %son %s\n", location->id, location->rsc_lh->id, ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""), node_name); break; case mon_output_html: case mon_output_cgi: node_name = get_node_display_name(node, mon_ops); fprintf(state->stream, "
  • %s prevents %s from running %son %s
    • \n", location->id, location->rsc_lh->id, ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""), node_name); break; case mon_output_xml: fprintf(state->stream, " \n", location->id, location->rsc_lh->id, node->details->uname, node->weight, ((location->role_filter == RSC_ROLE_MASTER)? "true" : "false")); break; default: break; } free(node_name); } /*! * \internal * \brief Print section for negative location constraints * * \param[in] stream File stream to display output to * \param[in] data_set Working set corresponding to CIB status to display */ static void print_neg_locations(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops, const char *prefix) { GListPtr gIter, gIter2; /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nNegative Location Constraints:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Negative Location Constraints

    \n
      \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } /* Print each ban */ for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) { pe__location_t *location = gIter->data; if (!g_str_has_prefix(location->id, prefix)) continue; for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) { node_t *node = (node_t *) gIter2->data; if (node->weight < 0) { print_ban(state, node, location, mon_ops); } } } /* Close section */ switch (state->output_format) { case mon_output_cgi: case mon_output_html: fprintf(state->stream, "
    \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print node attributes section * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state */ static void print_node_attributes(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops) { GListPtr gIter = NULL; /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nNode Attributes:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Node Attributes

    \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } /* Unpack all resource parameters (it would be more efficient to do this * only when needed for the first time in print_attr_msg()) */ for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { crm_mon_get_parameters(gIter->data, data_set); } /* Display each node's attributes */ for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { struct mon_attr_data data; data.state = state; data.node = (node_t *) gIter->data; if (data.node && data.node->details && data.node->details->online) { GList *attr_list = NULL; GHashTableIter iter; gpointer key, value; print_node_start(state, data.node, mon_ops); g_hash_table_iter_init(&iter, data.node->details->attrs); while (g_hash_table_iter_next (&iter, &key, &value)) { attr_list = append_attr_list(attr_list, key); } g_list_foreach(attr_list, print_node_attribute, &data); g_list_free(attr_list); print_node_end(state); } } /* Print section footer */ switch (state->output_format) { case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print header for cluster summary if needed * * \param[in] stream File stream to display output to */ static void print_cluster_summary_header(mon_state_t *state) { switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "

    Cluster Summary

    \n

    \n"); break; case mon_output_xml: fprintf(state->stream, "

    \n"); break; default: break; } } /*! * \internal * \brief Print footer for cluster summary if needed * * \param[in] stream File stream to display output to */ static void print_cluster_summary_footer(mon_state_t *state) { switch (state->output_format) { case mon_output_cgi: case mon_output_html: fprintf(state->stream, "

    \n"); break; case mon_output_xml: fprintf(state->stream, "     \n"); break; default: break; } } /*! * \internal * \brief Print times the display was last updated and CIB last changed * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state */ static void print_cluster_times(mon_state_t *state, pe_working_set_t *data_set) { const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN); const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER); const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT); const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG); switch (state->output_format) { case mon_output_plain: case mon_output_console: { const char *now_str = crm_now_string(NULL); print_as(state->output_format, "Last updated: %s", now_str ? now_str : "Could not determine current time"); print_as(state->output_format, (user || client || origin)? "\n" : "\t\t"); print_as(state->output_format, "Last change: %s", last_written ? last_written : ""); if (user) { print_as(state->output_format, " by %s", user); } if (client) { print_as(state->output_format, " via %s", client); } if (origin) { print_as(state->output_format, " on %s", origin); } print_as(state->output_format, "\n"); break; } case mon_output_html: case mon_output_cgi: { const char *now_str = crm_now_string(NULL); fprintf(state->stream, " Last updated: %s
    \n", now_str ? now_str : "Could not determine current time"); fprintf(state->stream, " Last change: %s", last_written ? last_written : ""); if (user) { fprintf(state->stream, " by %s", user); } if (client) { fprintf(state->stream, " via %s", client); } if (origin) { fprintf(state->stream, " on %s", origin); } fprintf(state->stream, "
    \n"); break; } case mon_output_xml: { const char *now_str = crm_now_string(NULL); fprintf(state->stream, " \n", now_str ? now_str : "Could not determine current time"); fprintf(state->stream, " \n", last_written ? last_written : "", user ? user : "", client ? client : "", origin ? origin : ""); break; } default: break; } } /*! * \internal * \brief Print cluster stack * * \param[in] stream File stream to display output to * \param[in] stack_s Stack name */ static void print_cluster_stack(mon_state_t *state, const char *stack_s) { switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "Stack: %s\n", stack_s); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " Stack: %s
    \n", stack_s); break; case mon_output_xml: fprintf(state->stream, " \n", stack_s); break; default: break; } } /*! * \internal * \brief Print current DC and its version * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state */ static void print_cluster_dc(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops) { node_t *dc = data_set->dc_node; xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']", data_set->input, LOG_DEBUG); const char *dc_version_s = dc_version? crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE) : NULL; const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM); char *dc_name = dc? get_node_display_name(dc, mon_ops) : NULL; switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "Current DC: "); if (dc) { print_as(state->output_format, "%s (version %s) - partition %s quorum\n", dc_name, (dc_version_s? dc_version_s : "unknown"), (crm_is_true(quorum) ? "with" : "WITHOUT")); } else { print_as(state->output_format, "NONE\n"); } break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " Current DC: "); if (dc) { fprintf(state->stream, "%s (version %s) - partition %s quorum", dc_name, (dc_version_s? dc_version_s : "unknown"), (crm_is_true(quorum)? "with" : "WITHOUT")); } else { fprintf(state->stream, "NONE"); } fprintf(state->stream, "
    \n"); break; case mon_output_xml: fprintf(state->stream, " stream, "present=\"true\" version=\"%s\" name=\"%s\" id=\"%s\" with_quorum=\"%s\"", (dc_version_s? dc_version_s : ""), dc->details->uname, dc->details->id, (crm_is_true(quorum) ? "true" : "false")); } else { fprintf(state->stream, "present=\"false\""); } fprintf(state->stream, " />\n"); break; default: break; } free(dc_name); } /*! * \internal * \brief Print counts of configured nodes and resources * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state * \param[in] stack_s Stack name */ static void print_cluster_counts(mon_state_t *state, pe_working_set_t *data_set, const char *stack_s) { int nnodes = g_list_length(data_set->nodes); int nresources = count_resources(data_set, NULL); switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\n%d node%s configured\n", nnodes, s_if_plural(nnodes)); print_as(state->output_format, "%d resource%s configured", nresources, s_if_plural(nresources)); if(data_set->disabled_resources || data_set->blocked_resources) { print_as(state->output_format, " ("); if (data_set->disabled_resources) { print_as(state->output_format, "%d DISABLED", data_set->disabled_resources); } if (data_set->disabled_resources && data_set->blocked_resources) { print_as(state->output_format, ", "); } if (data_set->blocked_resources) { print_as(state->output_format, "%d BLOCKED from starting due to failure", data_set->blocked_resources); } print_as(state->output_format, ")"); } print_as(state->output_format, "\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " %d node%s configured
    \n", nnodes, s_if_plural(nnodes)); fprintf(state->stream, " %d resource%s configured", nresources, s_if_plural(nresources)); if (data_set->disabled_resources || data_set->blocked_resources) { fprintf(state->stream, " ("); if (data_set->disabled_resources) { fprintf(state->stream, "%d DISABLED", data_set->disabled_resources); } if (data_set->disabled_resources && data_set->blocked_resources) { fprintf(state->stream, ", "); } if (data_set->blocked_resources) { fprintf(state->stream, "%d BLOCKED from starting due to failure", data_set->blocked_resources); } fprintf(state->stream, ")"); } fprintf(state->stream, "
    \n"); break; case mon_output_xml: fprintf(state->stream, " \n", g_list_length(data_set->nodes)); fprintf(state->stream, " \n", count_resources(data_set, NULL), data_set->disabled_resources, data_set->blocked_resources); break; default: break; } } /*! * \internal * \brief Print cluster-wide options * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state * * \note Currently this is only implemented for HTML and XML output, and * prints only a few options. If there is demand, more could be added. */ static void print_cluster_options(mon_state_t *state, pe_working_set_t *data_set) { switch (state->output_format) { case mon_output_plain: case mon_output_console: if (is_set(data_set->flags, pe_flag_maintenance_mode)) { print_as(state->output_format, "\n *** Resource management is DISABLED ***"); print_as(state->output_format, "\n The cluster will not attempt to start, stop or recover services"); print_as(state->output_format, "\n"); } break; case mon_output_html: fprintf(state->stream, "

    \n

    Config Options

    \n"); fprintf(state->stream, " \n"); fprintf(state->stream, " \n", is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled"); fprintf(state->stream, " \n", is_set(data_set->flags, pe_flag_symmetric_cluster)? "" : "a"); fprintf(state->stream, " \n"); fprintf(state->stream, " \n"); fprintf(state->stream, "
    STONITH of failed nodes%s
    Cluster is%ssymmetric
    No Quorum Policy"); switch (data_set->no_quorum_policy) { case no_quorum_freeze: fprintf(state->stream, "Freeze resources"); break; case no_quorum_stop: fprintf(state->stream, "Stop ALL resources"); break; case no_quorum_ignore: fprintf(state->stream, "Ignore"); break; case no_quorum_suicide: fprintf(state->stream, "Suicide"); break; } fprintf(state->stream, "
    Resource management"); if (is_set(data_set->flags, pe_flag_maintenance_mode)) { fprintf(state->stream, "DISABLED (the cluster will " "not attempt to start, stop or recover services)"); } else { fprintf(state->stream, "enabled"); } fprintf(state->stream, "
    \n

    \n"); break; case mon_output_xml: fprintf(state->stream, " stream, " stonith-enabled=\"%s\"", is_set(data_set->flags, pe_flag_stonith_enabled)? "true" : "false"); fprintf(state->stream, " symmetric-cluster=\"%s\"", is_set(data_set->flags, pe_flag_symmetric_cluster)? "true" : "false"); fprintf(state->stream, " no-quorum-policy=\""); switch (data_set->no_quorum_policy) { case no_quorum_freeze: fprintf(state->stream, "freeze"); break; case no_quorum_stop: fprintf(state->stream, "stop"); break; case no_quorum_ignore: fprintf(state->stream, "ignore"); break; case no_quorum_suicide: fprintf(state->stream, "suicide"); break; } fprintf(state->stream, "\""); fprintf(state->stream, " maintenance-mode=\"%s\"", is_set(data_set->flags, pe_flag_maintenance_mode)? "true" : "false"); fprintf(state->stream, " />\n"); break; default: break; } } /*! * \internal * \brief Print a summary of cluster-wide information * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state */ static void print_cluster_summary(mon_state_t *state, pe_working_set_t *data_set, unsigned int mon_ops, unsigned int show) { const char *stack_s = get_cluster_stack(data_set); gboolean header_printed = FALSE; if (show & mon_show_stack) { if (header_printed == FALSE) { print_cluster_summary_header(state); header_printed = TRUE; } print_cluster_stack(state, stack_s); } /* Always print DC if none, even if not requested */ if ((data_set->dc_node == NULL) || (show & mon_show_dc)) { if (header_printed == FALSE) { print_cluster_summary_header(state); header_printed = TRUE; } print_cluster_dc(state, data_set, mon_ops); } if (show & mon_show_times) { if (header_printed == FALSE) { print_cluster_summary_header(state); header_printed = TRUE; } print_cluster_times(state, data_set); } if (is_set(data_set->flags, pe_flag_maintenance_mode) || data_set->disabled_resources || data_set->blocked_resources || is_set(show, mon_show_count)) { if (header_printed == FALSE) { print_cluster_summary_header(state); header_printed = TRUE; } print_cluster_counts(state, data_set, stack_s); } /* There is not a separate option for showing cluster options, so show with * stack for now; a separate option could be added if there is demand */ if (show & mon_show_stack) { print_cluster_options(state, data_set); } if (header_printed) { print_cluster_summary_footer(state); } } /*! * \internal * \brief Print a failed action * * \param[in] stream File stream to display output to * \param[in] xml_op Root of XML tree describing failed action */ static void print_failed_action(mon_state_t *state, xmlNode *xml_op) { const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY); const char *op_key_attr = "op_key"; const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE); const char *node = crm_element_value(xml_op, XML_ATTR_UNAME); const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID); const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON); int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0"); int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0"); char *exit_reason_cleaned; /* If no op_key was given, use id instead */ if (op_key == NULL) { op_key = ID(xml_op); op_key_attr = "id"; } /* If no exit reason was given, use "none" */ if (exit_reason == NULL) { exit_reason = "none"; } /* Print common action information */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "* %s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'", op_key, node, services_ocf_exitcode_str(rc), rc, call, services_lrm_status_str(status), exit_reason); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "

  • %s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'", op_key, node, services_ocf_exitcode_str(rc), rc, call, services_lrm_status_str(status), exit_reason); break; case mon_output_xml: exit_reason_cleaned = crm_xml_escape(exit_reason); fprintf(state->stream, " stream, " exitstatus=\"%s\" exitreason=\"%s\" exitcode=\"%d\"", services_ocf_exitcode_str(rc), exit_reason_cleaned, rc); fprintf(state->stream, " call=\"%s\" status=\"%s\"", call, services_lrm_status_str(status)); free(exit_reason_cleaned); break; default: break; } /* If last change was given, print timing information as well */ if (last) { time_t run_at = crm_parse_int(last, "0"); char *run_at_s = ctime(&run_at); if (run_at_s) { run_at_s[24] = 0; /* Overwrite the newline */ } switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, ",\n last-rc-change='%s', queued=%sms, exec=%sms", run_at_s? run_at_s : "", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE), crm_element_value(xml_op, XML_RSC_OP_T_EXEC)); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, " last-rc-change='%s', queued=%sms, exec=%sms", run_at_s? run_at_s : "", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE), crm_element_value(xml_op, XML_RSC_OP_T_EXEC)); break; case mon_output_xml: fprintf(state->stream, " last-rc-change=\"%s\" queued=\"%s\" exec=\"%s\" interval=\"%u\" task=\"%s\"", run_at_s? run_at_s : "", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE), crm_element_value(xml_op, XML_RSC_OP_T_EXEC), crm_parse_ms(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS)), crm_element_value(xml_op, XML_LRM_ATTR_TASK)); break; default: break; } } /* End the action listing */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    • \n"); break; case mon_output_xml: fprintf(state->stream, " />\n"); break; default: break; } } /*! * \internal * \brief Print a section for failed actions * * \param[in] stream File stream to display output to * \param[in] data_set Working set of CIB state */ static void print_failed_actions(mon_state_t *state, pe_working_set_t *data_set) { xmlNode *xml_op = NULL; /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nFailed Resource Actions:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Failed Resource Actions

    \n
      \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } /* Print each failed action */ for (xml_op = __xml_first_child(data_set->failed); xml_op != NULL; xml_op = __xml_next(xml_op)) { print_failed_action(state, xml_op); } /* End section */ switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } /*! * \internal * \brief Print a stonith action * * \param[in] stream File stream to display output to * \param[in] event stonith event */ -static gboolean -is_later_succeeded(stonith_history_t *event, stonith_history_t *top_history) -{ - - gboolean ret = FALSE; - - for (stonith_history_t *prev_hp = top_history; prev_hp; prev_hp = prev_hp->next) { - if (prev_hp == event) { - break; - } - - if ((prev_hp->state == st_done) && - safe_str_eq(event->target, prev_hp->target) && - safe_str_eq(event->action, prev_hp->action) && - safe_str_eq(event->delegate, prev_hp->delegate) && - (event->completed < prev_hp->completed)) { - ret = TRUE; - break; - } - } - return ret; -} - static void print_stonith_action(mon_state_t *state, stonith_history_t *event, unsigned int mon_ops, stonith_history_t *top_history) { const char *action_s = stonith_action_str(event->action); char *run_at_s = ctime(&event->completed); if ((run_at_s) && (run_at_s[0] != 0)) { run_at_s[strlen(run_at_s)-1] = 0; /* Overwrite the newline */ } switch(state->output_format) { case mon_output_xml: fprintf(state->stream, " target, event->action); switch(event->state) { case st_done: fprintf(state->stream, " state=\"success\""); break; case st_failed: fprintf(state->stream, " state=\"failed\""); break; default: fprintf(state->stream, " state=\"pending\""); } fprintf(state->stream, " origin=\"%s\" client=\"%s\"", event->origin, event->client); if (event->delegate) { fprintf(state->stream, " delegate=\"%s\"", event->delegate); } switch(event->state) { case st_done: case st_failed: fprintf(state->stream, " completed=\"%s\"", run_at_s?run_at_s:""); break; default: break; } fprintf(state->stream, " />\n"); break; case mon_output_plain: case mon_output_console: switch(event->state) { case st_done: print_as(state->output_format, "* %s of %s successful: delegate=%s, client=%s, origin=%s,\n" " %s='%s'\n", action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, is_set(mon_ops, mon_op_fence_full_history) ? "completed" : "last-successful", run_at_s?run_at_s:""); break; case st_failed: print_as(state->output_format, "* %s of %s failed: delegate=%s, client=%s, origin=%s,\n" " %s='%s' %s\n", action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, is_set(mon_ops, mon_op_fence_full_history) ? "completed" : "last-failed", run_at_s?run_at_s:"", - is_later_succeeded(event, top_history) ? "(a later attempt succeeded)" : ""); + stonith__later_succeeded(event, top_history) ? "(a later attempt succeeded)" : ""); break; default: print_as(state->output_format, "* %s of %s pending: client=%s, origin=%s\n", action_s, event->target, event->client, event->origin); } break; case mon_output_html: case mon_output_cgi: switch(event->state) { case st_done: fprintf(state->stream, "
  • %s of %s successful: delegate=%s, " "client=%s, origin=%s, %s='%s'
    • \n", action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, is_set(mon_ops, mon_op_fence_full_history) ? "completed" : "last-successful", run_at_s?run_at_s:""); break; case st_failed: fprintf(state->stream, "
  • %s of %s failed: delegate=%s, " "client=%s, origin=%s, %s='%s' %s
    • \n", action_s, event->target, event->delegate ? event->delegate : "", event->client, event->origin, is_set(mon_ops, mon_op_fence_full_history) ? "completed" : "last-failed", run_at_s?run_at_s:"", - is_later_succeeded(event, top_history) ? "(a later attempt succeeded)" : ""); + stonith__later_succeeded(event, top_history) ? "(a later attempt succeeded)" : ""); break; default: fprintf(state->stream, "
  • %s of %s pending: client=%s, " "origin=%s
    • \n", action_s, event->target, event->client, event->origin); } break; default: /* no support for fence history for other formats so far */ break; } } /*! * \internal * \brief Print a section for failed stonith actions * * \param[in] stream File stream to display output to * \param[in] history List of stonith actions * */ static void print_failed_stonith_actions(mon_state_t *state, stonith_history_t *history, unsigned int mon_ops) { stonith_history_t *hp; for (hp = history; hp; hp = hp->next) { if (hp->state == st_failed) { break; } } if (!hp) { return; } /* Print section heading */ switch (state->output_format) { /* no need to take care of xml in here as xml gets full * history anyway */ case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nFailed Fencing Actions:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Failed Fencing Actions

    \n
      \n"); break; default: break; } /* Print each failed stonith action */ for (hp = history; hp; hp = hp->next) { if (hp->state == st_failed) { print_stonith_action(state, hp, mon_ops, history); } } /* End section */ switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n"); break; default: break; } } /*! * \internal * \brief Print pending stonith actions * * \param[in] stream File stream to display output to * \param[in] history List of stonith actions * */ static void print_stonith_pending(mon_state_t *state, stonith_history_t *history, unsigned int mon_ops) { /* xml-output always shows the full history * so we'll never have to show pending-actions * separately */ if (history && (history->state != st_failed) && (history->state != st_done)) { stonith_history_t *hp; /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nPending Fencing Actions:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Pending Fencing Actions

    \n
      \n"); break; default: break; } for (hp = history; hp; hp = hp->next) { if ((hp->state == st_failed) || (hp->state == st_done)) { break; } print_stonith_action(state, hp, mon_ops, NULL); } /* End section */ switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n"); break; default: break; } } } /*! * \internal * \brief Print a section for stonith-history * * \param[in] stream File stream to display output to * \param[in] history List of stonith actions * */ static void print_stonith_history(mon_state_t *state, stonith_history_t *history, unsigned int mon_ops) { stonith_history_t *hp; /* Print section heading */ switch (state->output_format) { case mon_output_plain: case mon_output_console: print_as(state->output_format, "\nFencing History:\n"); break; case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n

    Fencing History

    \n
      \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } for (hp = history; hp; hp = hp->next) { if ((hp->state != st_failed) || (state->output_format == mon_output_xml)) { print_stonith_action(state, hp, mon_ops, NULL); } } /* End section */ switch (state->output_format) { case mon_output_html: case mon_output_cgi: fprintf(state->stream, "
    \n"); break; case mon_output_xml: fprintf(state->stream, " \n"); break; default: break; } } void print_status(mon_state_t *state, pe_working_set_t *data_set, stonith_history_t *stonith_history, unsigned int mon_ops, unsigned int show, const char *prefix) { GListPtr gIter = NULL; int print_opts = get_resource_display_options(mon_ops, state->output_format); /* space-separated lists of node names */ char *online_nodes = NULL; char *online_remote_nodes = NULL; char *online_guest_nodes = NULL; char *offline_nodes = NULL; char *offline_remote_nodes = NULL; if (state->output_format == mon_output_console) { blank_screen(); } print_cluster_summary(state, data_set, mon_ops, show); print_as(state->output_format, "\n"); /* Gather node information (and print if in bad state or grouping by node) */ for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; const char *node_mode = NULL; char *node_name = get_node_display_name(node, mon_ops); /* Get node mode */ if (node->details->unclean) { if (node->details->online) { node_mode = "UNCLEAN (online)"; } else if (node->details->pending) { node_mode = "UNCLEAN (pending)"; } else { node_mode = "UNCLEAN (offline)"; } } else if (node->details->pending) { node_mode = "pending"; } else if (node->details->standby_onfail && node->details->online) { node_mode = "standby (on-fail)"; } else if (node->details->standby) { if (node->details->online) { if (node->details->running_rsc) { node_mode = "standby (with active resources)"; } else { node_mode = "standby"; } } else { node_mode = "OFFLINE (standby)"; } } else if (node->details->maintenance) { if (node->details->online) { node_mode = "maintenance"; } else { node_mode = "OFFLINE (maintenance)"; } } else if (node->details->online) { node_mode = "online"; if (is_not_set(mon_ops, mon_op_group_by_node)) { if (pe__is_guest_node(node)) { online_guest_nodes = add_list_element(online_guest_nodes, node_name); } else if (pe__is_remote_node(node)) { online_remote_nodes = add_list_element(online_remote_nodes, node_name); } else { online_nodes = add_list_element(online_nodes, node_name); } free(node_name); continue; } } else { node_mode = "OFFLINE"; if (is_not_set(mon_ops, mon_op_group_by_node)) { if (pe__is_remote_node(node)) { offline_remote_nodes = add_list_element(offline_remote_nodes, node_name); } else if (pe__is_guest_node(node)) { /* ignore offline guest nodes */ } else { offline_nodes = add_list_element(offline_nodes, node_name); } free(node_name); continue; } } /* If we get here, node is in bad state, or we're grouping by node */ /* Print the node name and status */ if (pe__is_guest_node(node)) { print_as(state->output_format, "Guest"); } else if (pe__is_remote_node(node)) { print_as(state->output_format, "Remote"); } print_as(state->output_format, "Node %s: %s\n", node_name, node_mode); /* If we're grouping by node, print its resources */ if (is_set(mon_ops, mon_op_group_by_node)) { if (is_set(mon_ops, mon_op_print_brief)) { print_rscs_brief(node->details->running_rsc, "\t", print_opts | pe_print_rsconly, state->stream, FALSE); } else { GListPtr gIter2 = NULL; for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) { resource_t *rsc = (resource_t *) gIter2->data; rsc->fns->print(rsc, "\t", print_opts | pe_print_rsconly, state->stream); } } } free(node_name); } /* If we're not grouping by node, summarize nodes by status */ if (online_nodes) { print_as(state->output_format, "Online: [%s ]\n", online_nodes); free(online_nodes); } if (offline_nodes) { print_as(state->output_format, "OFFLINE: [%s ]\n", offline_nodes); free(offline_nodes); } if (online_remote_nodes) { print_as(state->output_format, "RemoteOnline: [%s ]\n", online_remote_nodes); free(online_remote_nodes); } if (offline_remote_nodes) { print_as(state->output_format, "RemoteOFFLINE: [%s ]\n", offline_remote_nodes); free(offline_remote_nodes); } if (online_guest_nodes) { print_as(state->output_format, "GuestOnline: [%s ]\n", online_guest_nodes); free(online_guest_nodes); } /* Print resources section, if needed */ print_resources(state, data_set, print_opts, mon_ops); /* print Node Attributes section if requested */ if (show & mon_show_attributes) { print_node_attributes(state, data_set, mon_ops); } /* If requested, print resource operations (which includes failcounts) * or just failcounts */ if (show & (mon_show_operations | mon_show_failcounts)) { print_node_summary(state, data_set, ((show & mon_show_operations)? TRUE : FALSE), mon_ops); } /* If there were any failed actions, print them */ if (xml_has_children(data_set->failed)) { print_failed_actions(state, data_set); } /* Print failed stonith actions */ if (is_set(mon_ops, mon_op_fence_history)) { print_failed_stonith_actions(state, stonith_history, mon_ops); } /* Print tickets if requested */ if (show & mon_show_tickets) { print_cluster_tickets(state, data_set); } /* Print negative location constraints if requested */ if (show & mon_show_bans) { print_neg_locations(state, data_set, mon_ops, prefix); } /* Print stonith history */ if (is_set(mon_ops, mon_op_fence_history)) { if (show & mon_show_fence_history) { print_stonith_history(state, stonith_history, mon_ops); } else { print_stonith_pending(state, stonith_history, mon_ops); } } #if CURSES_ENABLED if (state->output_format == mon_output_console) { refresh(); } #endif } void print_xml_status(mon_state_t *state, pe_working_set_t *data_set, stonith_history_t *stonith_history, unsigned int mon_ops, unsigned int show, const char *prefix) { GListPtr gIter = NULL; int print_opts = get_resource_display_options(mon_ops, state->output_format); fprintf(state->stream, "\n"); fprintf(state->stream, "\n", VERSION); print_cluster_summary(state, data_set, mon_ops, show); /*** NODES ***/ fprintf(state->stream, " \n"); for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; const char *node_type = "unknown"; switch (node->details->type) { case node_member: node_type = "member"; break; case node_remote: node_type = "remote"; break; case node_ping: node_type = "ping"; break; } fprintf(state->stream, " details->uname); fprintf(state->stream, "id=\"%s\" ", node->details->id); fprintf(state->stream, "online=\"%s\" ", node->details->online ? "true" : "false"); fprintf(state->stream, "standby=\"%s\" ", node->details->standby ? "true" : "false"); fprintf(state->stream, "standby_onfail=\"%s\" ", node->details->standby_onfail ? "true" : "false"); fprintf(state->stream, "maintenance=\"%s\" ", node->details->maintenance ? "true" : "false"); fprintf(state->stream, "pending=\"%s\" ", node->details->pending ? "true" : "false"); fprintf(state->stream, "unclean=\"%s\" ", node->details->unclean ? "true" : "false"); fprintf(state->stream, "shutdown=\"%s\" ", node->details->shutdown ? "true" : "false"); fprintf(state->stream, "expected_up=\"%s\" ", node->details->expected_up ? "true" : "false"); fprintf(state->stream, "is_dc=\"%s\" ", node->details->is_dc ? "true" : "false"); fprintf(state->stream, "resources_running=\"%d\" ", g_list_length(node->details->running_rsc)); fprintf(state->stream, "type=\"%s\" ", node_type); if (pe__is_guest_node(node)) { fprintf(state->stream, "id_as_resource=\"%s\" ", node->details->remote_rsc->container->id); } if (is_set(mon_ops, mon_op_group_by_node)) { GListPtr lpc2 = NULL; fprintf(state->stream, ">\n"); for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) { resource_t *rsc = (resource_t *) lpc2->data; rsc->fns->print(rsc, " ", print_opts | pe_print_rsconly, state->stream); } fprintf(state->stream, " \n"); } else { fprintf(state->stream, "/>\n"); } } fprintf(state->stream, " \n"); /* Print resources section, if needed */ print_resources(state, data_set, print_opts, mon_ops); /* print Node Attributes section if requested */ if (show & mon_show_attributes) { print_node_attributes(state, data_set, mon_ops); } /* If requested, print resource operations (which includes failcounts) * or just failcounts */ if (show & (mon_show_operations | mon_show_failcounts)) { print_node_summary(state, data_set, ((show & mon_show_operations)? TRUE : FALSE), mon_ops); } /* If there were any failed actions, print them */ if (xml_has_children(data_set->failed)) { print_failed_actions(state, data_set); } /* Print stonith history */ if (is_set(mon_ops, mon_op_fence_history)) { print_stonith_history(state, stonith_history, mon_ops); } /* Print tickets if requested */ if (show & mon_show_tickets) { print_cluster_tickets(state, data_set); } /* Print negative location constraints if requested */ if (show & mon_show_bans) { print_neg_locations(state, data_set, mon_ops, prefix); } fprintf(state->stream, "\n"); fflush(state->stream); } int print_html_status(mon_state_t *state, pe_working_set_t *data_set, const char *filename, stonith_history_t *stonith_history, unsigned int mon_ops, unsigned int show, const char *prefix, unsigned int reconnect_msec) { GListPtr gIter = NULL; char *filename_tmp = NULL; int print_opts = get_resource_display_options(mon_ops, state->output_format); if (state->output_format == mon_output_cgi) { fprintf(state->stream, "Content-Type: text/html\n\n"); } else { filename_tmp = crm_concat(filename, "tmp", '.'); state->stream = fopen(filename_tmp, "w"); if (state->stream == NULL) { crm_perror(LOG_ERR, "Cannot open %s for writing", filename_tmp); free(filename_tmp); return -1; } } fprintf(state->stream, "\n"); fprintf(state->stream, " \n"); fprintf(state->stream, " Cluster status\n"); fprintf(state->stream, " \n", reconnect_msec / 1000); fprintf(state->stream, " \n"); fprintf(state->stream, "\n"); print_cluster_summary(state, data_set, mon_ops, show); /*** NODE LIST ***/ fprintf(state->stream, "
    \n

    Node List

    \n"); fprintf(state->stream, "
      \n"); for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; char *node_name = get_node_display_name(node, mon_ops); fprintf(state->stream, "
    • Node: %s: ", node_name); if (node->details->standby_onfail && node->details->online) { fprintf(state->stream, "standby (on-fail)\n"); } else if (node->details->standby && node->details->online) { fprintf(state->stream, "standby%s\n", node->details->running_rsc?" (with active resources)":""); } else if (node->details->standby) { fprintf(state->stream, "OFFLINE (standby)\n"); } else if (node->details->maintenance && node->details->online) { fprintf(state->stream, "maintenance\n"); } else if (node->details->maintenance) { fprintf(state->stream, "OFFLINE (maintenance)\n"); } else if (node->details->online) { fprintf(state->stream, "online\n"); } else { fprintf(state->stream, "OFFLINE\n"); } if (is_set(mon_ops, mon_op_print_brief) && is_set(mon_ops, mon_op_group_by_node)) { fprintf(state->stream, "
        \n"); print_rscs_brief(node->details->running_rsc, NULL, print_opts | pe_print_rsconly, state->stream, FALSE); fprintf(state->stream, "
      \n"); } else if (is_set(mon_ops, mon_op_group_by_node)) { GListPtr lpc2 = NULL; fprintf(state->stream, "
        \n"); for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) { resource_t *rsc = (resource_t *) lpc2->data; fprintf(state->stream, "
      • "); rsc->fns->print(rsc, NULL, print_opts | pe_print_rsconly, state->stream); fprintf(state->stream, "
        • \n"); } fprintf(state->stream, "
      \n"); } fprintf(state->stream, "
      • \n"); free(node_name); } fprintf(state->stream, "
    \n"); /* Print resources section, if needed */ print_resources(state, data_set, print_opts, mon_ops); /* print Node Attributes section if requested */ if (show & mon_show_attributes) { print_node_attributes(state, data_set, mon_ops); } /* If requested, print resource operations (which includes failcounts) * or just failcounts */ if (show & (mon_show_operations | mon_show_failcounts)) { print_node_summary(state, data_set, ((show & mon_show_operations)? TRUE : FALSE), mon_ops); } /* If there were any failed actions, print them */ if (xml_has_children(data_set->failed)) { print_failed_actions(state, data_set); } /* Print failed stonith actions */ if (is_set(mon_ops, mon_op_fence_history)) { print_failed_stonith_actions(state, stonith_history, mon_ops); } /* Print stonith history */ if (is_set(mon_ops, mon_op_fence_history)) { if (show & mon_show_fence_history) { print_stonith_history(state, stonith_history, mon_ops); } else { print_stonith_pending(state, stonith_history, mon_ops); } } /* Print tickets if requested */ if (show & mon_show_tickets) { print_cluster_tickets(state, data_set); } /* Print negative location constraints if requested */ if (show & mon_show_bans) { print_neg_locations(state, data_set, mon_ops, prefix); } fprintf(state->stream, "\n"); fprintf(state->stream, "\n"); fflush(state->stream); if (state->output_format != mon_output_cgi) { if (rename(filename_tmp, filename) != 0) { crm_perror(LOG_ERR, "Unable to rename %s->%s", filename_tmp, filename); } free(filename_tmp); } return 0; } diff --git a/tools/stonith_admin.c b/tools/stonith_admin.c index 841bd51091..b0e40e74dc 100644 --- a/tools/stonith_admin.c +++ b/tools/stonith_admin.c @@ -1,837 +1,837 @@ /* * Copyright 2009-2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SUMMARY "stonith_admin - Access the Pacemaker fencing API" char action = 0; struct { gboolean as_nodeid; gboolean broadcast; gboolean cleanup; gboolean installed; gboolean metadata; gboolean registered; gboolean validate_cfg; stonith_key_value_t *devices; stonith_key_value_t *params; int fence_level; int timeout ; int tolerance; char *agent; char *confirm_host; char *fence_host; char *history; char *last_fenced; char *query; char *reboot_host; char *register_dev; char *register_level; char *targets; char *terminate; char *unfence_host; char *unregister_dev; char *unregister_level; } options = { .timeout = 120 }; gboolean add_env_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); gboolean add_stonith_device(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); gboolean add_stonith_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); gboolean add_tolerance(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); gboolean set_tag(const gchar *option_name, const gchar *optarg, gpointer data, GError **error); #define INDENT " " /* *INDENT-OFF* */ static GOptionEntry defn_entries[] = { { "register", 'R', 0, G_OPTION_ARG_STRING, &options.register_dev, "Register the named stonith device. Requires: --agent.\n" INDENT "Optional: --option, --env-option.", "DEVICE" }, { "deregister", 'D', 0, G_OPTION_ARG_STRING, &options.unregister_dev, "De-register the named stonith device.", "DEVICE" }, { "register-level", 'r', 0, G_OPTION_ARG_STRING, &options.register_level, "Register a stonith level for the named target,\n" INDENT "specified as one of NAME, @PATTERN, or ATTR=VALUE.\n" INDENT "Requires: --index and one or more --device entries.", "TARGET" }, { "deregister-level", 'd', 0, G_OPTION_ARG_STRING, &options.unregister_level, "Unregister a stonith level for the named target,\n" INDENT "specified as for --register-level. Requires: --index", "TARGET" }, { NULL } }; static GOptionEntry query_entries[] = { { "list", 'l', 0, G_OPTION_ARG_STRING, &options.terminate, "List devices that can terminate the specified host.\n" INDENT "Optional: --timeout", "HOST" }, { "list-registered", 'L', 0, G_OPTION_ARG_NONE, &options.registered, "List all registered devices. Optional: --timeout.", NULL }, { "list-installed", 'I', 0, G_OPTION_ARG_NONE, &options.installed, "List all installed devices. Optional: --timeout.", NULL }, { "list-targets", 's', 0, G_OPTION_ARG_STRING, &options.targets, "List the targets that can be fenced by the\n" INDENT "named device. Optional: --timeout.", "DEVICE" }, { "metadata", 'M', 0, G_OPTION_ARG_NONE, &options.metadata, "Show agent metadata. Requires: --agent.\n" INDENT "Optional: --timeout.", NULL }, { "query", 'Q', 0, G_OPTION_ARG_STRING, &options.query, "Check the named device's status. Optional: --timeout.", "DEVICE" }, { "history", 'H', 0, G_OPTION_ARG_STRING, &options.history, "Show last successful fencing operation for named node\n" INDENT "(or '*' for all nodes). Optional: --timeout, --cleanup,\n" INDENT "--quiet (show only the operation's epoch timestamp),\n" INDENT "--verbose (show all recorded and pending operations),\n" INDENT "--broadcast (update history from all nodes available).", "NODE" }, { "last", 'h', 0, G_OPTION_ARG_STRING, &options.last_fenced, "Indicate when the named node was last fenced.\n" INDENT "Optional: --as-node-id.", "NODE" }, { "validate", 'K', 0, G_OPTION_ARG_NONE, &options.validate_cfg, "Validate a fence device configuration.\n" INDENT "Requires: --agent. Optional: --option, --env-option,\n" INDENT "--quiet (print no output, only return status).", NULL }, { NULL } }; static GOptionEntry fence_entries[] = { { "fence", 'F', 0, G_OPTION_ARG_STRING, &options.fence_host, "Fence named host. Optional: --timeout, --tolerance.", "HOST" }, { "unfence", 'U', 0, G_OPTION_ARG_STRING, &options.unfence_host, "Unfence named host. Optional: --timeout, --tolerance.", "HOST" }, { "reboot", 'B', 0, G_OPTION_ARG_STRING, &options.reboot_host, "Reboot named host. Optional: --timeout, --tolerance.", "HOST" }, { "confirm", 'C', 0, G_OPTION_ARG_STRING, &options.confirm_host, "Tell cluster that named host is now safely down.", "HOST", }, { NULL } }; static GOptionEntry addl_entries[] = { { "cleanup", 'c', 0, G_OPTION_ARG_NONE, &options.cleanup, "Cleanup wherever appropriate. Requires --history.", NULL }, { "broadcast", 'b', 0, G_OPTION_ARG_NONE, &options.broadcast, "Broadcast wherever appropriate.", NULL }, { "agent", 'a', 0, G_OPTION_ARG_STRING, &options.agent, "The agent to use (for example, fence_xvm;\n" INDENT "with --register, --metadata, --validate).", "AGENT" }, { "option", 'o', 0, G_OPTION_ARG_CALLBACK, add_stonith_params, "Specify a device configuration parameter as NAME=VALUE\n" INDENT "(may be specified multiple times; with --register,\n" INDENT "--validate).", "PARAM" }, { "env-option", 'e', 0, G_OPTION_ARG_CALLBACK, add_env_params, "Specify a device configuration parameter with the\n" INDENT "specified name, using the value of the\n" INDENT "environment variable of the same name prefixed with\n" INDENT "OCF_RESKEY_ (may be specified multiple times;\n" INDENT "with --register, --validate).", "PARAM" }, { "tag", 'T', 0, G_OPTION_ARG_CALLBACK, set_tag, "Identify fencing operations in logs with the specified\n" INDENT "tag; useful when multiple entities might invoke\n" INDENT "stonith_admin (used with most commands).", "TAG" }, { "device", 'v', 0, G_OPTION_ARG_CALLBACK, add_stonith_device, "Device ID (with --register-level, device to associate with\n" INDENT "a given host and level; may be specified multiple times)" #if SUPPORT_CIBSECRETS "\n" INDENT "(with --validate, name to use to load CIB secrets)" #endif ".", "DEVICE" }, { "index", 'i', 0, G_OPTION_ARG_INT, &options.fence_level, "The stonith level (1-9) (with --register-level,\n" INDENT "--deregister-level).", "LEVEL" }, { "timeout", 't', 0, G_OPTION_ARG_INT, &options.timeout, "Operation timeout in seconds (default 120;\n" INDENT "used with most commands).", "SECONDS" }, { "as-node-id", 'n', 0, G_OPTION_ARG_NONE, &options.as_nodeid, "(Advanced) The supplied node is the corosync node ID\n" INDENT "(with --last).", NULL }, { "tolerance", 0, 0, G_OPTION_ARG_CALLBACK, add_tolerance, "(Advanced) Do nothing if an equivalent --fence request\n" INDENT "succeeded less than this many seconds earlier\n" INDENT "(with --fence, --unfence, --reboot).", "SECONDS" }, { NULL } }; /* *INDENT-ON* */ static pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_HTML, PCMK__SUPPORTED_FORMAT_TEXT, PCMK__SUPPORTED_FORMAT_XML, { NULL, NULL, NULL } }; static int st_opts = st_opt_sync_call | st_opt_allow_suicide; static GMainLoop *mainloop = NULL; struct { stonith_t *st; const char *target; const char *action; char *name; int timeout; int tolerance; int rc; } async_fence_data; gboolean add_env_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) { char *key = crm_concat("OCF_RESKEY", optarg, '_'); const char *env = getenv(key); gboolean retval = TRUE; if (env == NULL) { crm_err("Invalid option: -e %s", optarg); g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM, "Invalid option: -e %s", optarg); retval = FALSE; } else { crm_info("Got: '%s'='%s'", optarg, env); options.params = stonith_key_value_add(options.params, optarg, env); } free(key); return retval; } gboolean add_stonith_device(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) { options.devices = stonith_key_value_add(options.devices, NULL, optarg); return TRUE; } gboolean add_tolerance(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) { options.tolerance = crm_get_msec(optarg) / 1000; return TRUE; } gboolean add_stonith_params(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) { char *name = NULL; char *value = NULL; int rc = 0; gboolean retval = TRUE; crm_info("Scanning: -o %s", optarg); rc = pcmk_scan_nvpair(optarg, &name, &value); if (rc != 2) { crm_err("Invalid option: -o %s: %s", optarg, pcmk_strerror(rc)); g_set_error(error, G_OPTION_ERROR, rc, "Invalid option: -o %s: %s", optarg, pcmk_strerror(rc)); retval = FALSE; } else { crm_info("Got: '%s'='%s'", name, value); options.params = stonith_key_value_add(options.params, name, value); } free(name); free(value); return retval; } gboolean set_tag(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) { free(async_fence_data.name); async_fence_data.name = crm_strdup_printf("%s.%s", crm_system_name, optarg); return TRUE; } static void notify_callback(stonith_t * st, stonith_event_t * e) { if (e->result != pcmk_ok) { return; } if (safe_str_eq(async_fence_data.target, e->target) && safe_str_eq(async_fence_data.action, e->action)) { async_fence_data.rc = e->result; g_main_loop_quit(mainloop); } } static void fence_callback(stonith_t * stonith, stonith_callback_data_t * data) { async_fence_data.rc = data->rc; g_main_loop_quit(mainloop); } static gboolean async_fence_helper(gpointer user_data) { stonith_t *st = async_fence_data.st; int call_id = 0; int rc = stonith_api_connect_retry(st, async_fence_data.name, 10); if (rc != pcmk_ok) { fprintf(stderr, "Could not connect to fencer: %s\n", pcmk_strerror(rc)); g_main_loop_quit(mainloop); return TRUE; } st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, notify_callback); call_id = st->cmds->fence(st, st_opt_allow_suicide, async_fence_data.target, async_fence_data.action, async_fence_data.timeout, async_fence_data.tolerance); if (call_id < 0) { g_main_loop_quit(mainloop); return TRUE; } st->cmds->register_callback(st, call_id, async_fence_data.timeout, st_opt_timeout_updates, NULL, "callback", fence_callback); return TRUE; } static int mainloop_fencing(stonith_t * st, const char *target, const char *action, int timeout, int tolerance) { crm_trigger_t *trig; async_fence_data.st = st; async_fence_data.target = target; async_fence_data.action = action; async_fence_data.timeout = timeout; async_fence_data.tolerance = tolerance; async_fence_data.rc = -1; trig = mainloop_add_trigger(G_PRIORITY_HIGH, async_fence_helper, NULL); mainloop_set_trigger(trig); mainloop = g_main_loop_new(NULL, FALSE); g_main_loop_run(mainloop); return async_fence_data.rc; } static int handle_level(stonith_t *st, char *target, int fence_level, stonith_key_value_t *devices, bool added) { char *node = NULL; char *pattern = NULL; char *name = NULL; char *value = NULL; if (target == NULL) { // Not really possible, but makes static analysis happy return -EINVAL; } /* Determine if targeting by attribute, node name pattern or node name */ value = strchr(target, '='); if (value != NULL) { name = target; *value++ = '\0'; } else if (*target == '@') { pattern = target + 1; } else { node = target; } /* Register or unregister level as appropriate */ if (added) { return st->cmds->register_level_full(st, st_opts, node, pattern, name, value, fence_level, devices); } return st->cmds->remove_level_full(st, st_opts, node, pattern, name, value, fence_level); } static int handle_history(stonith_t *st, const char *target, int timeout, int quiet, int verbose, int cleanup, int broadcast, pcmk__output_t *out) { stonith_history_t *history = NULL, *hp, *latest = NULL; int rc = 0; if (!quiet) { if (cleanup) { out->info(out, "cleaning up fencing-history%s%s", target ? " for node " : "", target ? target : ""); } if (broadcast) { out->info(out, "gather fencing-history from all nodes"); } } rc = st->cmds->history(st, st_opts | (cleanup?st_opt_cleanup:0) | (broadcast?st_opt_broadcast:0), (safe_str_eq(target, "*")? NULL : target), &history, timeout); out->begin_list(out, "Fencing history", "event", "events"); for (hp = history; hp; hp = hp->next) { if (hp->state == st_done) { latest = hp; } if (quiet || !verbose) { continue; } - out->message(out, "stonith-event", hp, 1); + out->message(out, "stonith-event", hp, 1, history); } if (latest) { if (quiet && out->supports_quiet) { out->info(out, "%lld", (long long) latest->completed); } else if (!verbose) { // already printed if verbose - out->message(out, "stonith-event", latest, 0); + out->message(out, "stonith-event", latest, 0, NULL); } } out->end_list(out); stonith_history_free(history); return rc; } static int validate(stonith_t *st, const char *agent, const char *id, stonith_key_value_t *params, int timeout, int quiet, pcmk__output_t *out) { int rc = 1; char *output = NULL; char *error_output = NULL; rc = st->cmds->validate(st, st_opt_sync_call, id, NULL, agent, params, timeout, &output, &error_output); if (quiet) { return rc; } out->message(out, "validate", agent, id, output, error_output, rc); return rc; } static void show_last_fenced(pcmk__output_t *out, const char *target) { time_t when = 0; if (target == NULL) { // Not really possible, but makes static analysis happy return; } if (options.as_nodeid) { uint32_t nodeid = atol(target); when = stonith_api_time(nodeid, NULL, FALSE); } else { when = stonith_api_time(0, target, FALSE); } out->message(out, "last-fenced", target, when); } static GOptionContext * build_arg_context(pcmk__common_args_t *args) { GOptionContext *context = NULL; GOptionEntry extra_prog_entries[] = { { "quiet", 'q', 0, G_OPTION_ARG_NONE, &(args->quiet), "Be less descriptive in output.", NULL }, { NULL } }; context = pcmk__build_arg_context(args, "text (default), html, xml"); /* Add the -q option, which cannot be part of the globally supported options * because some tools use that flag for something else. */ pcmk__add_main_args(context, extra_prog_entries); pcmk__add_arg_group(context, "definition", "Device Definition Commands:", "Show device definition help", defn_entries); pcmk__add_arg_group(context, "queries", "Queries:", "Show query help", query_entries); pcmk__add_arg_group(context, "fence", "Fencing Commands:", "Show fence help", fence_entries); pcmk__add_arg_group(context, "additional", "Additional Options:", "Show additional options", addl_entries); return context; } int main(int argc, char **argv) { int rc = 0; bool no_connect = false; bool required_agent = false; char *target = NULL; char *lists = NULL; const char *device = NULL; crm_exit_t exit_code = CRM_EX_OK; stonith_t *st = NULL; stonith_key_value_t *dIter = NULL; pcmk__output_t *out = NULL; pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); GError *error = NULL; GOptionContext *context = NULL; gchar **processed_args = NULL; context = build_arg_context(args); pcmk__register_formats(context, formats); crm_log_cli_init("stonith_admin"); async_fence_data.name = strdup(crm_system_name); processed_args = pcmk__cmdline_preproc(argc, argv, "adehilorstvBCDFHQRTU"); if (!g_option_context_parse_strv(context, &processed_args, &error)) { fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message); } for (int i = 0; i < args->verbosity; i++) { crm_bump_log_level(argc, argv); } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if (rc != 0) { fprintf(stderr, "Error creating output format %s: %s\n", args->output_ty, pcmk_strerror(rc)); exit_code = CRM_EX_ERROR; goto done; } stonith__register_messages(out); if (args->version) { out->version(out, false); goto done; } if (options.validate_cfg) { required_agent = true; no_connect = true; action = 'K'; } if (options.installed) { no_connect = true; action = 'I'; } if (options.registered) { action = 'L'; } if (options.register_dev != NULL) { required_agent = true; action = 'R'; device = options.register_dev; } if (options.query != NULL) { action = 'Q'; device = options.query; } if (options.unregister_dev != NULL) { action = 'D'; device = options.unregister_dev; } if (options.targets != NULL) { action = 's'; device = options.targets; } if (options.terminate != NULL) { action = 'L'; target = options.terminate; } if (options.metadata) { no_connect = true; required_agent = true; action = 'M'; } if (options.reboot_host != NULL) { no_connect = true; action = 'B'; target = options.reboot_host; crm_log_args(argc, argv); } if (options.fence_host != NULL) { no_connect = true; action = 'F'; target = options.fence_host; crm_log_args(argc, argv); } if (options.unfence_host != NULL) { no_connect = true; action = 'U'; target = options.unfence_host; crm_log_args(argc, argv); } if (options.confirm_host != NULL) { action = 'C'; target = options.confirm_host; crm_log_args(argc, argv); } if (options.last_fenced != NULL) { action = 'h'; target = options.last_fenced; } if (options.history != NULL) { action = 'H'; target = options.history; } if (options.register_level != NULL) { action = 'r'; target = options.register_level; } if (options.unregister_level != NULL) { action = 'd'; target = options.unregister_level; } if (optind > argc || action == 0) { out->err(out, "%s", g_option_context_get_help(context, TRUE, NULL)); exit_code = CRM_EX_USAGE; goto done; } if (required_agent && options.agent == NULL) { out->err(out, "Please specify an agent to query using -a,--agent [value]"); out->err(out, "%s", g_option_context_get_help(context, TRUE, NULL)); exit_code = CRM_EX_USAGE; goto done; } st = stonith_api_new(); if (st == NULL) { rc = -ENOMEM; } else if (!no_connect) { rc = st->cmds->connect(st, async_fence_data.name, NULL); } if (rc < 0) { out->err(out, "Could not connect to fencer: %s", pcmk_strerror(rc)); exit_code = CRM_EX_DISCONNECT; goto done; } switch (action) { case 'I': rc = st->cmds->list_agents(st, st_opt_sync_call, NULL, &options.devices, options.timeout); if (rc < 0) { out->err(out, "Failed to list installed devices: %s", pcmk_strerror(rc)); break; } out->begin_list(out, "Installed fence devices", "fence device", "fence devices"); for (dIter = options.devices; dIter; dIter = dIter->next) { out->list_item(out, "device", dIter->value); } out->end_list(out); rc = 0; stonith_key_value_freeall(options.devices, 1, 1); break; case 'L': rc = st->cmds->query(st, st_opts, target, &options.devices, options.timeout); if (rc < 0) { out->err(out, "Failed to list registered devices: %s", pcmk_strerror(rc)); break; } out->begin_list(out, "Registered fence devices", "fence device", "fence devices"); for (dIter = options.devices; dIter; dIter = dIter->next) { out->list_item(out, "device", dIter->value); } out->end_list(out); rc = 0; stonith_key_value_freeall(options.devices, 1, 1); break; case 'Q': rc = st->cmds->monitor(st, st_opts, device, options.timeout); if (rc < 0) { rc = st->cmds->list(st, st_opts, device, NULL, options.timeout); } break; case 's': rc = st->cmds->list(st, st_opts, device, &lists, options.timeout); if (rc == 0) { GList *targets = stonith__parse_targets(lists); out->begin_list(out, "Fence targets", "fence target", "fence targets"); while (targets != NULL) { out->list_item(out, NULL, (const char *) targets->data); targets = targets->next; } out->end_list(out); free(lists); } else if (rc != 0) { out->err(out, "List command returned error. rc : %d", rc); } break; case 'R': rc = st->cmds->register_device(st, st_opts, device, NULL, options.agent, options.params); break; case 'D': rc = st->cmds->remove_device(st, st_opts, device); break; case 'd': case 'r': rc = handle_level(st, target, options.fence_level, options.devices, action == 'r'); break; case 'M': { char *buffer = NULL; rc = st->cmds->metadata(st, st_opt_sync_call, options.agent, NULL, &buffer, options.timeout); if (rc == pcmk_ok) { out->output_xml(out, "metadata", buffer); } free(buffer); } break; case 'C': rc = st->cmds->confirm(st, st_opts, target); break; case 'B': rc = mainloop_fencing(st, target, "reboot", options.timeout, options.tolerance); break; case 'F': rc = mainloop_fencing(st, target, "off", options.timeout, options.tolerance); break; case 'U': rc = mainloop_fencing(st, target, "on", options.timeout, options.tolerance); break; case 'h': show_last_fenced(out, target); break; case 'H': rc = handle_history(st, target, options.timeout, args->quiet, args->verbosity, options.cleanup, options.broadcast, out); break; case 'K': device = (options.devices ? options.devices->key : NULL); rc = validate(st, options.agent, device, options.params, options.timeout, args->quiet, out); break; } crm_info("Command returned: %s (%d)", pcmk_strerror(rc), rc); exit_code = crm_errno2exit(rc); done: g_strfreev(processed_args); g_option_context_free(context); if (out != NULL) { out->finish(out, exit_code, true, NULL); pcmk__output_free(out); } free(async_fence_data.name); stonith_key_value_freeall(options.params, 1, 1); if (st != NULL) { st->cmds->disconnect(st); stonith_api_delete(st); } return exit_code; }