diff --git a/fencing/commands.c b/fencing/commands.c index b0ada51887..8808e2ed37 100644 --- a/fencing/commands.c +++ b/fencing/commands.c @@ -1,540 +1,554 @@ /* * Copyright (C) 2009 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define FE_AGENT_FORK -2 #define FE_AGENT_ERROR -3 GHashTable *device_list = NULL; int invoke_device(stonith_device_t *device, const char *action, const char *port, char **output); static void append_arg( gpointer key, gpointer value, gpointer user_data) { int len = 3; /* =, \n, \0 */ int last = 0; char **args = user_data; if(strstr(key, "pcmk-")) { return; } len += strlen(key); len += strlen(value); if(*args != NULL) { last = strlen(*args); } crm_realloc(*args, last+len); sprintf((*args)+last, "%s=%s\n", (char *)key, (char *)value); } static char *make_args(GHashTable *args) { char *arg_list = NULL; g_hash_table_foreach(args, append_arg, &arg_list); crm_debug_3("Calculated: %s", arg_list); return arg_list; } /* Borrowed from libfence */ static int run_agent(char *agent, GHashTable *arg_hash, int *agent_result, char **output) { char *args = make_args(arg_hash); int pid, status, len, rc = -1; int pr_fd, pw_fd; /* parent read/write file descriptors */ int cr_fd, cw_fd; /* child read/write file descriptors */ int fd1[2]; int fd2[2]; cr_fd = cw_fd = pr_fd = pw_fd = -1; if (args == NULL || agent == NULL) goto fail; len = strlen(args); if (pipe(fd1)) goto fail; pr_fd = fd1[0]; cw_fd = fd1[1]; if (pipe(fd2)) goto fail; cr_fd = fd2[0]; pw_fd = fd2[1]; pid = fork(); if (pid < 0) { *agent_result = FE_AGENT_FORK; goto fail; } if (pid) { /* parent */ int ret; fcntl(pr_fd, F_SETFL, fcntl(pr_fd, F_GETFL, 0) | O_NONBLOCK); do { ret = write(pw_fd, args, len); } while (ret < 0 && errno == EINTR); if (ret != len) goto fail; close(pw_fd); waitpid(pid, &status, 0); if(output != NULL) { len = 0; do { char buf[500]; ret = read(pr_fd, buf, 500); if(ret > 0) { buf[ret] = 0; crm_realloc(*output, len + ret + 1); sprintf((*output)+len, "%s", buf); len += ret; } } while (ret < 0 && errno == EINTR); } crm_info("%d %d", WIFEXITED(status), WEXITSTATUS(status)); *agent_result = FE_AGENT_ERROR; if (WIFEXITED(status)) { *agent_result = -WEXITSTATUS(status); rc = 0; } } else { /* child */ close(1); if (dup(cw_fd) < 0) goto fail; close(2); if (dup(cw_fd) < 0) goto fail; close(0); if (dup(cr_fd) < 0) goto fail; /* keep cw_fd open so parent can report all errors. */ close(pr_fd); close(cr_fd); close(pw_fd); execlp(agent, agent, NULL); exit(EXIT_FAILURE); } fail: crm_free(args); close(pr_fd); close(cw_fd); close(cr_fd); close(pw_fd); return rc; } static void free_device(gpointer data) { stonith_device_t *device = data; g_hash_table_destroy(device->params); crm_free(device->namespace); crm_free(device->agent); crm_free(device->id); crm_free(device); } static void build_port_aliases(stonith_device_t *device) { char *name = NULL; char *value = NULL; int last = 0, lpc = 0, max = 0; const char *portmap = g_hash_table_lookup(device->params, "pcmk-portmap"); if(portmap == NULL) { return; } max = strlen(portmap); for(; lpc < max; lpc++) { if(portmap[lpc] == 0) { break; } else if(isalpha(portmap[lpc])) { /* keep going */ } else if(portmap[lpc] == '=') { crm_malloc0(name, 1 + lpc - last); strncpy(name, portmap + last, lpc - last); last = lpc + 1; } else if(name && isspace(portmap[lpc])) { crm_malloc0(value, 1 + lpc - last); strncpy(value, portmap + last, lpc - last); last = lpc + 1; crm_info("Adding alias '%s'='%s' for %s", name, value, device->id); g_hash_table_replace(device->aliases, name, value); value=NULL; name=NULL; } else if(isspace(portmap[lpc])) { last = lpc; } } } static int stonith_device_register(xmlNode *msg) { xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR); stonith_device_t *device = NULL; crm_malloc0(device, sizeof(stonith_device_t)); device->id = crm_element_value_copy(dev, XML_ATTR_ID); device->agent = crm_element_value_copy(dev, "agent"); device->namespace = crm_element_value_copy(dev, "namespace"); device->params = xml2list(dev); device->aliases = g_hash_table_new_full(g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); build_port_aliases(device); g_hash_table_insert(device_list, device->id, device); crm_info("Added '%s' to the device list (%d active devices)", device->id, g_hash_table_size(device_list)); return stonith_ok; } static int stonith_device_remove(xmlNode *msg) { xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR); const char *id = crm_element_value(dev, XML_ATTR_ID); if(g_hash_table_remove(device_list, id)) { crm_info("Removed '%s' from the device list (%d active devices)", id, g_hash_table_size(device_list)); } else { crm_info("Device '%s' not found (%d active devices)", id, g_hash_table_size(device_list)); } return stonith_ok; } static const char *get_device_port(stonith_device_t *dev, const char *host) { time_t now; char *alias = NULL; if(host == NULL) { return NULL; } now = time(NULL); alias = g_hash_table_lookup(dev->aliases, host); if(dev->targets == NULL || dev->targets_age + 300 < now) { int rc = stonith_ok; char *output = NULL; crm_free(dev->targets); dev->targets = NULL; rc = invoke_device(dev, "list", NULL, &output); crm_info("Port list for %s: %d", dev->id, rc); if(rc == 0) { crm_info("Refreshing port list for %s", dev->id); dev->targets = output; dev->targets_age = now; } else { crm_info("Disabling port list queries for %s", dev->id); dev->targets_age = -1; } } /* See if portmap is defined and look up the translated name */ if(alias) { if(dev->targets && strstr(dev->targets, alias)) { return alias; } else if(dev->targets == NULL) { return alias; } } if(dev->targets && strstr(dev->targets, host)) { return host; } return NULL; } int invoke_device(stonith_device_t *device, const char *action, const char *port, char **output) { int rc = 0; const char *device_port = get_device_port(device, port); if(port && device_port) { g_hash_table_replace(device->params, crm_strdup("port"), crm_strdup(device_port)); } else if(port) { crm_err("Unknown or unhandled port '%s' for device '%s'", port, device->id); return -1; } crm_info("Calling '%s' with action '%s'%s%s", device->id, action, port?" on port ":"", port?port:""); g_hash_table_replace(device->params, crm_strdup("option"), crm_strdup(action)); if(run_agent(device->agent, device->params, &rc, output) < 0) { crm_err("Operation %s on %s failed (%d): %s", action, device->id, rc, *output); } else { crm_info("Operation %s on %s passed: %s", action, device->id, *output); } g_hash_table_remove(device->params, "port"); return rc; } static int stonith_device_action(xmlNode *msg, char **output) { int rc = stonith_ok; xmlNode *dev = get_xpath_object("//@"F_STONITH_DEVICE, msg, LOG_ERR); const char *id = crm_element_value(dev, F_STONITH_DEVICE); const char *action = crm_element_value(dev, F_STONITH_ACTION); const char *port = crm_element_value(dev, F_STONITH_PORT); stonith_device_t *device = NULL; if(id) { crm_info("Looking for '%s'", id); device = g_hash_table_lookup(device_list, id); } if(device) { rc = invoke_device(device, action, port, output); } else { crm_err("Device %s not found", id); } return rc; } struct device_search_s { const char *host; GListPtr capable; }; static void search_devices( gpointer key, gpointer value, gpointer user_data) { stonith_device_t *dev = value; struct device_search_s *search = user_data; if(get_device_port(value, search->host)) { crm_debug_4("Device '%s' can fence '%s'", dev->id, search->host); search->capable = g_list_append(search->capable, value); } else { crm_debug_3("Device '%s' cannot fence '%s'", dev->id, search->host); } } static int stonith_query(xmlNode *msg, xmlNode **list) { struct device_search_s search; xmlNode *dev = get_xpath_object("//@target", msg, LOG_ERR); search.capable = NULL; search.host = crm_element_value(dev, "target"); g_hash_table_foreach(device_list, search_devices, &search); crm_info("Found %d matching devices for '%s'", g_list_length(search.capable), search.host); /* Pack the results into data */ if(list) { *list = create_xml_node(NULL, __FUNCTION__); crm_xml_add_int(*list, "st-available-devices", g_list_length(search.capable)); slist_iter(device, stonith_device_t, search.capable, lpc, dev = create_xml_node(*list, F_STONITH_DEVICE); crm_xml_add(dev, XML_ATTR_ID, device->id); crm_xml_add(dev, "namespace", device->namespace); crm_xml_add(dev, "agent", device->agent); ); } return g_list_length(search.capable); } static int stonith_fence(xmlNode *msg, const char *action) { struct device_search_s search; xmlNode *dev = get_xpath_object("//@target", msg, LOG_ERR); search.capable = NULL; search.host = crm_element_value(dev, "target"); g_hash_table_foreach(device_list, search_devices, &search); crm_info("Found %d matching devices for '%s'", g_list_length(search.capable), search.host); slist_iter(dev, stonith_device_t, search.capable, lpc, int rc = 0; char *output = NULL; const char *port = get_device_port(dev, search.host); CRM_CHECK(port != NULL, continue); g_hash_table_replace(dev->params, crm_strdup("option"), crm_strdup(action)); g_hash_table_replace(dev->params, crm_strdup("port"), crm_strdup(port)); if(run_agent(dev->agent, dev->params, &rc, &output) == 0) { crm_info("Terminated host '%s' with device '%s'", search.host, dev->id); crm_free(output); return stonith_ok; } else { crm_err("Termination of host '%s' with device '%s' failed: %s", search.host, dev->id, output); } crm_free(output); ); return -1; } static xmlNode * stonith_construct_reply(xmlNode *request, char *output, xmlNode *data, int rc) { int lpc = 0; xmlNode *reply = NULL; const char *name = NULL; const char *value = NULL; const char *names[] = { F_STONITH_OPERATION, F_STONITH_CALLID, F_STONITH_CLIENTID, F_STONITH_CALLOPTS }; crm_debug_4("Creating a basic reply"); reply = create_xml_node(NULL, "stonith-reply"); crm_xml_add(reply, F_TYPE, T_STONITH); for(lpc = 0; lpc < DIMOF(names); lpc++) { name = names[lpc]; value = crm_element_value(request, name); crm_xml_add(reply, name, value); } crm_xml_add_int(reply, F_STONITH_RC, rc); crm_xml_add(reply, "st_output", output); if(data != NULL) { crm_debug_4("Attaching reply output"); add_message_xml(reply, F_STONITH_CALLDATA, data); } return reply; } void stonith_command(stonith_client_t *client, xmlNode *request, gboolean remote) { int rc = stonith_ok; int call_options = 0; gboolean done = TRUE; xmlNode *reply = NULL; xmlNode *data = NULL; char *output = NULL; const char *op = crm_element_value(request, F_STONITH_OPERATION); const char *client_id = crm_element_value(request, F_STONITH_CLIENTID); crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options); if(device_list == NULL) { device_list = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, free_device); } crm_info("Processing %s from %s", op, client->name); if(crm_str_eq(op, CRM_OP_REGISTER, TRUE)) { return; } else if(crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) { - /* Update the notify filters for this client */ - int on_off = 0; - crm_element_value_int(request, F_STONITH_NOTIFY_ACTIVATE, &on_off); - - crm_debug("Setting callbacks for %s (%s): %s", - client->name, client->id, on_off?"on":"off"); - client->flags = on_off; + const char *flag_name = NULL; + + flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE); + if(flag_name) { + crm_debug("Setting %s callbacks for %s (%s): ON", + flag_name, client->name, client->id); + client->flags |= get_stonith_flag(flag_name); + } + + flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE); + if(flag_name) { + crm_debug("Setting %s callbacks for %s (%s): off", + flag_name, client->name, client->id); + client->flags |= get_stonith_flag(flag_name); + } return; } else if(crm_str_eq(op, STONITH_OP_DEVICE_ADD, TRUE)) { rc = stonith_device_register(request); + do_stonith_notify(call_options, op, rc, request, T_STONITH_NOTIFY_DEVICE_ADD); } else if(crm_str_eq(op, STONITH_OP_DEVICE_DEL, TRUE)) { rc = stonith_device_remove(request); + do_stonith_notify(call_options, op, rc, request, T_STONITH_NOTIFY_DEVICE_DEL); + } else if(crm_str_eq(op, STONITH_OP_EXEC, TRUE)) { rc = stonith_device_action(request, &output); } else if(crm_str_eq(op, STONITH_OP_FENCE, TRUE)) { rc = stonith_fence(request, "off"); + do_stonith_notify(call_options, op, rc, request, T_STONITH_NOTIFY_FENCE); + if(rc < 0) { stonith_query(request, &data); } } else if(crm_str_eq(op, STONITH_OP_UNFENCE, TRUE)) { rc = stonith_fence(request, "on"); + do_stonith_notify(call_options, op, rc, request, T_STONITH_NOTIFY_UNFENCE); } else if(crm_str_eq(op, STONITH_OP_QUERY, TRUE)) { rc = stonith_query(request, &data); } if(done) { reply = stonith_construct_reply(request, output, data, rc); - do_local_notify(reply, client_id, call_options & stonith_sync_call, remote); + do_local_reply(reply, client_id, call_options & stonith_sync_call, remote); free_xml(reply); } crm_free(output); free_xml(data); } diff --git a/fencing/internal.h b/fencing/internal.h index dba50eeab6..1e554d9cf5 100644 --- a/fencing/internal.h +++ b/fencing/internal.h @@ -1,32 +1,37 @@ typedef struct stonith_device_s { char *id; char *agent; char *namespace; char *targets; time_t targets_age; GHashTable *params; GHashTable *aliases; } stonith_device_t; typedef struct stonith_client_s { char *id; char *name; char *callback_id; const char *channel_name; IPC_Channel *channel; GCHSource *source; long long flags; } stonith_client_t; extern void stonith_command(stonith_client_t *client, xmlNode *op_request, gboolean remote); -extern void do_local_notify(xmlNode *notify_src, const char *client_id, - gboolean sync_reply, gboolean from_peer); +extern void do_local_reply( + xmlNode *notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer); + +extern long long get_stonith_flag(const char *name); + +extern void do_stonith_notify( + int options, const char *op, enum stonith_errors result, xmlNode *data, const char *type); diff --git a/fencing/main.c b/fencing/main.c index 412aebad8c..f4325282e7 100644 --- a/fencing/main.c +++ b/fencing/main.c @@ -1,530 +1,611 @@ /* * Copyright (C) 2009 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include char *channel1 = NULL; char *channel2 = NULL; char *stonith_our_uname = NULL; GMainLoop *mainloop = NULL; GHashTable *client_list = NULL; gboolean stonith_shutdown_flag = FALSE; #if SUPPORT_HEARTBEAT ll_cluster_t *hb_conn = NULL; #endif static gboolean stonith_client_disconnect( IPC_Channel *channel, stonith_client_t *stonith_client) { if (channel == NULL) { CRM_DEV_ASSERT(stonith_client == NULL); } else if (stonith_client == NULL) { crm_err("No client"); } else { CRM_DEV_ASSERT(channel->ch_status != IPC_CONNECT); crm_debug_2("Cleaning up after client disconnect: %s/%s/%s", crm_str(stonith_client->name), stonith_client->channel_name, stonith_client->id); if(stonith_client->id != NULL) { if(!g_hash_table_remove(client_list, stonith_client->id)) { crm_err("Client %s not found in the hashtable", stonith_client->name); } } } return FALSE; } static gboolean stonith_client_callback(IPC_Channel *channel, gpointer user_data) { int lpc = 0; const char *value = NULL; xmlNode *request = NULL; gboolean keep_channel = TRUE; stonith_client_t *stonith_client = user_data; CRM_CHECK(stonith_client != NULL, crm_err("Invalid client"); return FALSE); CRM_CHECK(stonith_client->id != NULL, crm_err("Invalid client: %p", stonith_client); return FALSE); if(IPC_ISRCONN(channel) && channel->ops->is_message_pending(channel)) { lpc++; request = xmlfromIPC(channel, MAX_IPC_DELAY); if (request == NULL) { goto bail; } if(stonith_client->name == NULL) { value = crm_element_value(request, F_STONITH_CLIENTNAME); if(value == NULL) { stonith_client->name = crm_itoa(channel->farside_pid); } else { stonith_client->name = crm_strdup(value); } } crm_xml_add(request, F_STONITH_CLIENTID, stonith_client->id); crm_xml_add(request, F_STONITH_CLIENTNAME, stonith_client->name); if(stonith_client->callback_id == NULL) { value = crm_element_value(request, F_STONITH_CALLBACK_TOKEN); if(value != NULL) { stonith_client->callback_id = crm_strdup(value); } else { stonith_client->callback_id = crm_strdup(stonith_client->id); } } crm_log_xml(LOG_MSG, "Client[inbound]", request); stonith_command(stonith_client, request, FALSE); free_xml(request); } bail: if(channel->ch_status != IPC_CONNECT) { crm_debug_2("Client disconnected"); keep_channel = stonith_client_disconnect(channel, stonith_client); } return keep_channel; } static void stonith_client_destroy(gpointer user_data) { stonith_client_t *stonith_client = user_data; if(stonith_client == NULL) { crm_debug_4("Destroying %p", user_data); return; } if(stonith_client->source != NULL) { crm_debug_4("Deleting %s (%p) from mainloop", stonith_client->name, stonith_client->source); G_main_del_IPC_Channel(stonith_client->source); stonith_client->source = NULL; } crm_debug_3("Destroying %s (%p)", stonith_client->name, user_data); crm_free(stonith_client->name); crm_free(stonith_client->callback_id); crm_free(stonith_client->id); crm_free(stonith_client); crm_debug_4("Freed the cib client"); return; } static gboolean stonith_client_connect(IPC_Channel *channel, gpointer user_data) { cl_uuid_t client_id; xmlNode *reg_msg = NULL; stonith_client_t *new_client = NULL; char uuid_str[UU_UNPARSE_SIZEOF]; const char *channel_name = user_data; crm_debug_3("Connecting channel"); CRM_CHECK(channel_name != NULL, return FALSE); if (channel == NULL) { crm_err("Channel was NULL"); return FALSE; } else if (channel->ch_status != IPC_CONNECT) { crm_err("Channel was disconnected"); return FALSE; } else if(stonith_shutdown_flag) { crm_info("Ignoring new client [%d] during shutdown", channel->farside_pid); return FALSE; } crm_malloc0(new_client, sizeof(stonith_client_t)); new_client->channel = channel; new_client->channel_name = channel_name; crm_debug_3("Created channel %p for channel %s", new_client, new_client->channel_name); channel->ops->set_recv_qlen(channel, 1024); channel->ops->set_send_qlen(channel, 1024); new_client->source = G_main_add_IPC_Channel( G_PRIORITY_DEFAULT, channel, FALSE, stonith_client_callback, new_client, stonith_client_destroy); crm_debug_3("Channel %s connected for client %s", new_client->channel_name, new_client->id); cl_uuid_generate(&client_id); cl_uuid_unparse(&client_id, uuid_str); CRM_CHECK(new_client->id == NULL, crm_free(new_client->id)); new_client->id = crm_strdup(uuid_str); /* make sure we can find ourselves later for sync calls * redirected to the master instance */ g_hash_table_insert(client_list, new_client->id, new_client); reg_msg = create_xml_node(NULL, "callback"); crm_xml_add(reg_msg, F_STONITH_OPERATION, CRM_OP_REGISTER); crm_xml_add(reg_msg, F_STONITH_CLIENTID, new_client->id); send_ipc_message(channel, reg_msg); free_xml(reg_msg); return TRUE; } static void stonith_peer_callback(xmlNode * msg, void* private_data) { crm_log_xml(LOG_MSG, "Peer[inbound]", msg); } static void stonith_peer_hb_callback(HA_Message * msg, void* private_data) { xmlNode *xml = convert_ha_message(NULL, msg, __FUNCTION__); stonith_peer_callback(xml, private_data); free_xml(xml); } #if SUPPORT_AIS static gboolean stonith_peer_ais_callback( AIS_Message *wrapper, char *data, int sender) { xmlNode *xml = NULL; if(wrapper->header.id == crm_class_cluster) { xml = string2xml(data); if(xml == NULL) { goto bail; } crm_xml_add(xml, F_ORIG, wrapper->sender.uname); crm_xml_add_int(xml, F_SEQ, wrapper->id); stonith_peer_callback(xml, NULL); } free_xml(xml); return TRUE; bail: crm_err("Invalid XML: '%.120s'", data); return TRUE; } static void stonith_peer_ais_destroy(gpointer user_data) { crm_err("AIS connection terminated"); ais_fd_sync = -1; exit(1); } #endif static void stonith_peer_hb_destroy(gpointer user_data) { if(stonith_shutdown_flag) { crm_info("Heartbeat disconnection complete... exiting"); } else { crm_err("Heartbeat connection lost! Exiting."); } crm_info("Exiting..."); if (mainloop != NULL && g_main_is_running(mainloop)) { g_main_quit(mainloop); } else { exit(LSB_EXIT_OK); } } static int send_via_callback_channel(xmlNode *msg, const char *token) { stonith_client_t *hash_client = NULL; enum stonith_errors rc = stonith_ok; crm_debug_3("Delivering msg %p to client %s", msg, token); if(token == NULL) { crm_err("No client id token, cant send message"); if(rc == stonith_ok) { rc = -1; } } else if(msg == NULL) { crm_err("No message to send"); rc = -1; } else { /* A client that left before we could reply is not really * _our_ error. Warn instead. */ hash_client = g_hash_table_lookup(client_list, token); if(hash_client == NULL) { crm_warn("Cannot find client for token %s", token); rc = -1; } else if (crm_str_eq(hash_client->channel_name, "remote", FALSE)) { /* just hope it's alive */ } else if(hash_client->channel == NULL) { crm_err("Cannot find channel for client %s", token); rc = -1; } } if(rc == stonith_ok) { crm_debug_3("Delivering reply to client %s (%s)", token, hash_client->channel_name); if(send_ipc_message(hash_client->channel, msg) == FALSE) { crm_warn("Delivery of reply to client %s/%s failed", hash_client->name, token); rc = -1; } } return rc; } -void do_local_notify(xmlNode *notify_src, const char *client_id, +void do_local_reply(xmlNode *notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer) { /* send callback to originating child */ stonith_client_t *client_obj = NULL; enum stonith_errors local_rc = stonith_ok; crm_debug_2("Performing notification"); if(client_id != NULL) { client_obj = g_hash_table_lookup(client_list, client_id); } else { crm_debug_2("No client to sent the response to." " F_STONITH_CLIENTID not set."); } crm_debug_3("Sending callback to request originator"); if(client_obj == NULL) { - local_rc = -123; + local_rc = -1; } else { const char *client_id = client_obj->callback_id; crm_debug_2("Sending %ssync response to %s %s", sync_reply?"":"an a-", client_obj->name, from_peer?"(originator of delegated request)":""); if(sync_reply) { client_id = client_obj->id; } local_rc = send_via_callback_channel(notify_src, client_id); } if(local_rc != stonith_ok && client_obj != NULL) { crm_warn("%sSync reply to %s failed: %s", sync_reply?"":"A-", client_obj?client_obj->name:"", stonith_error2string(local_rc)); } } +long long get_stonith_flag(const char *name) +{ + if(safe_str_eq(name, T_STONITH_NOTIFY_FENCE)) { + return 0x01; + + } else if(safe_str_eq(name, T_STONITH_NOTIFY_UNFENCE)) { + return 0x02; + + } else if(safe_str_eq(name, T_STONITH_NOTIFY_DEVICE_ADD)) { + return 0x04; + + } else if(safe_str_eq(name, T_STONITH_NOTIFY_DEVICE_DEL)) { + return 0x10; + } + return 0; +} + +static void +stonith_notify_client(gpointer key, gpointer value, gpointer user_data) +{ + + IPC_Channel *ipc_client = NULL; + xmlNode *update_msg = user_data; + stonith_client_t *client = value; + const char *type = NULL; + + CRM_CHECK(client != NULL, return); + CRM_CHECK(update_msg != NULL, return); + + type = crm_element_value(update_msg, F_SUBTYPE); + CRM_CHECK(type != NULL, return); + + if(client == NULL) { + crm_warn("Skipping NULL client"); + return; + + } else if(client->channel == NULL) { + crm_warn("Skipping client with NULL channel"); + return; + + } else if(client->name == NULL) { + crm_debug_2("Skipping unnammed client / comamnd channel"); + return; + } + + ipc_client = client->channel; + if(client->flags & get_stonith_flag(type)) { + crm_info("Sending %s-notification to client %s/%s", type, client->name, client->id); + if(ipc_client->send_queue->current_qlen >= ipc_client->send_queue->max_qlen) { + /* We never want the STONITH to exit because our client is slow */ + crm_crit("%s-notification of client %s/%s failed - queue saturated", + type, client->name, client->id); + + } else if(send_ipc_message(ipc_client, update_msg) == FALSE) { + crm_warn("%s-Notification of client %s/%s failed", + type, client->name, client->id); + } + } +} + +void +do_stonith_notify( + int options, const char *op, enum stonith_errors result, xmlNode *data, const char *type) +{ + xmlNode *update_msg = create_xml_node(NULL, "notify"); + + crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY); + crm_xml_add(update_msg, F_SUBTYPE, type); + crm_xml_add(update_msg, F_STONITH_OPERATION, op); + crm_xml_add_int(update_msg, F_STONITH_RC, result); + + if(data != NULL) { + add_message_xml(update_msg, F_STONITH_CALLDATA, data); + } + + crm_debug_3("Notifying clients"); + g_hash_table_foreach(client_list, stonith_notify_client, update_msg); + free_xml(update_msg); + crm_debug_3("Notify complete"); +} + static void stonith_shutdown(int nsig) { stonith_shutdown_flag = TRUE; crm_info("Terminating with %d clients", g_hash_table_size(client_list)); stonith_client_disconnect(NULL, NULL); } static void stonith_cleanup(void) { crm_peer_destroy(); g_hash_table_destroy(client_list); crm_free(stonith_our_uname); #if HAVE_LIBXML2 xmlCleanupParser(); #endif crm_free(channel1); } static struct crm_option long_options[] = { {"stand-alone", 0, 0, 's'}, {"verbose", 0, 0, 'V'}, {"version", 0, 0, '$'}, {"help", 0, 0, '?'}, {0, 0, 0, 0} }; int main(int argc, char ** argv) { int flag; int rc = 0; int argerr = 0; int option_index = 0; gboolean stand_alone = FALSE; crm_log_init("stonith-ng", LOG_INFO, TRUE, TRUE, argc, argv); crm_set_options("V?s$", "mode [options]", long_options, "Provides a summary of cluster's current state." "\n\nOutputs varying levels of detail in a number of different formats.\n"); mainloop_add_signal(SIGTERM, stonith_shutdown); /* EnableProcLogging(); */ set_sigchld_proctrack(G_PRIORITY_HIGH,DEFAULT_MAXDISPATCHTIME); crm_peer_init(); client_list = g_hash_table_new(g_str_hash, g_str_equal); while (1) { flag = crm_get_option(argc, argv, &option_index); if (flag == -1) break; switch(flag) { case 'V': alter_debug(DEBUG_INC); cl_log_enable_stderr(1); break; case 's': stand_alone = TRUE; cl_log_enable_stderr(1); break; case '$': case '?': crm_help(flag, LSB_EXIT_OK); break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { crm_help('?', LSB_EXIT_GENERIC); } if(stand_alone == FALSE) { void *dispatch = stonith_peer_hb_callback; void *destroy = stonith_peer_hb_destroy; if(is_openais_cluster()) { #if SUPPORT_AIS destroy = stonith_peer_ais_destroy; dispatch = stonith_peer_ais_callback; #endif } if(crm_cluster_connect(&stonith_our_uname, NULL, dispatch, destroy, #if SUPPORT_HEARTBEAT &hb_conn #else NULL #endif ) == FALSE){ crm_crit("Cannot sign in to the cluster... terminating"); exit(100); } } else { stonith_our_uname = crm_strdup("localhost"); } channel1 = crm_strdup(stonith_channel); rc = init_server_ipc_comms( channel1, stonith_client_connect, default_ipc_connection_destroy); channel2 = crm_strdup(stonith_channel_callback); rc = init_server_ipc_comms( channel2, stonith_client_connect, default_ipc_connection_destroy); if(rc == 0) { /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); g_main_run(mainloop); } else { crm_err("Couldnt start all communication channels, exiting."); } stonith_cleanup(); #if SUPPORT_HEARTBEAT if(hb_conn) { hb_conn->llc_ops->delete(hb_conn); } #endif crm_info("Done"); return rc; } diff --git a/fencing/test.c b/fencing/test.c index 71772a4ee5..0b6b23482f 100644 --- a/fencing/test.c +++ b/fencing/test.c @@ -1,156 +1,177 @@ /* * Copyright (C) 2009 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct crm_option long_options[] = { {"verbose", 0, 0, 'V'}, {"version", 0, 0, '$'}, {"help", 0, 0, '?'}, {0, 0, 0, 0} }; int st_opts = stonith_sync_call; +GMainLoop *mainloop = NULL; + +static void st_callback(const char *event, xmlNode *msg) +{ + crm_log_xml_notice(msg, event); +} + +static gboolean timeout_handler(gpointer data) +{ + g_main_quit(mainloop); + return FALSE; +} int main(int argc, char ** argv) { int flag; int rc = 0; int argerr = 0; int option_index = 0; stonith_t *st = NULL; GHashTable *hash = NULL; crm_log_init("stonith-test", LOG_INFO, TRUE, TRUE, argc, argv); crm_set_options("V?$", "mode [options]", long_options, "Provides a summary of cluster's current state." "\n\nOutputs varying levels of detail in a number of different formats.\n"); while (1) { flag = crm_get_option(argc, argv, &option_index); if (flag == -1) break; switch(flag) { case 'V': alter_debug(DEBUG_INC); cl_log_enable_stderr(1); break; case '$': case '?': crm_help(flag, LSB_EXIT_OK); break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { crm_help('?', LSB_EXIT_GENERIC); } hash = g_hash_table_new(g_str_hash, g_str_equal); g_hash_table_insert(hash, crm_strdup("ipaddr"), crm_strdup("localhost")); g_hash_table_insert(hash, crm_strdup("pcmk-portmap"), crm_strdup("some-host=pcmk-1 pcmk-3=3,4")); g_hash_table_insert(hash, crm_strdup("login"), crm_strdup("root")); g_hash_table_insert(hash, crm_strdup("identity_file"), crm_strdup("/root/.ssh/id_dsa")); crm_debug("Create"); st = stonith_api_new(); rc = st->cmds->connect(st, crm_system_name, NULL, NULL); crm_debug("Connect: %d", rc); + rc = st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, st_callback); + rc = st->cmds->register_notification(st, T_STONITH_NOTIFY_UNFENCE, st_callback); + rc = st->cmds->register_device(st, st_opts, "test-id", "stonith-ng", "fence_virsh", hash); crm_debug("Register: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "list", NULL, 10); crm_debug("List: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "monitor", NULL, 10); crm_debug("Monitor: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-2", 10); crm_debug("Status pcmk-2: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10); crm_debug("Status pcmk-1: %d", rc); rc = st->cmds->fence(st, st_opts, "unknown-host", 10); crm_debug("Fence unknown-host: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10); crm_debug("Status pcmk-1: %d", rc); rc = st->cmds->fence(st, st_opts, "pcmk-1", 10); crm_debug("Fence pcmk-1: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10); crm_debug("Status pcmk-1: %d", rc); rc = st->cmds->unfence(st, st_opts, "pcmk-1", 10); crm_debug("Unfence pcmk-1: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "status", "pcmk-1", 10); crm_debug("Status pcmk-1: %d", rc); rc = st->cmds->fence(st, st_opts, "some-host", 10); crm_debug("Fence alias: %d", rc); rc = st->cmds->call(st, st_opts, "test-id", "status", "some-host", 10); crm_debug("Status alias: %d", rc); rc = st->cmds->remove_device(st, st_opts, "test-id"); crm_debug("Remove test-id: %d", rc); + + crm_debug("Drain the notifications..."); + mainloop = g_main_new(FALSE); + crm_info("Starting %s mainloop", crm_system_name); + g_timeout_add(5*1000, timeout_handler, NULL); + + g_main_run(mainloop); - sleep(5); rc = st->cmds->disconnect(st); crm_debug("Disconnect: %d", rc); crm_debug("Destroy"); stonith_api_delete(st); return rc; } diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 1e054ac9e4..7663133c3a 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1,1094 +1,1097 @@ /* * Copyright (c) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * */ #include #include #include #include #include #include #include #include #include #include #include GHashTable *stonith_op_callback_table = NULL; typedef struct stonith_private_s { char *token; IPC_Channel *command_channel; IPC_Channel *callback_channel; GCHSource *callback_source; void (*op_callback)(const xmlNode *msg, int call_id, int rc, xmlNode *output, void *data); } stonith_private_t; typedef struct stonith_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*callback)(const char *event, xmlNode *msg); } stonith_notify_client_t; typedef struct stonith_callback_client_s { void (*callback)(const xmlNode*, int, int, xmlNode*, void*); const char *id; void *user_data; gboolean only_success; struct timer_rec_s *timer; } stonith_callback_client_t; struct timer_rec_s { int call_id; int timeout; guint ref; stonith_t *stonith; }; typedef enum stonith_errors (*stonith_op_t)( const char *, int, const char *, xmlNode *, xmlNode*, xmlNode*, xmlNode**, xmlNode**); gboolean stonith_dispatch(IPC_Channel *channel, gpointer user_data); void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc); xmlNode *stonith_create_op( int call_id, const char *token, const char *op, xmlNode *data, int call_options); int stonith_send_command( stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options); static void stonith_connection_destroy(gpointer user_data) { stonith_t *stonith = user_data; stonith->state = stonith_disconnected; /* TODO: Provide a notification */ /* g_list_foreach(stonith->notify_list, stonith_send_notification, msg); */ } static int stonith_api_register_device( stonith_t *stonith, int call_options, const char *id, const char *namespace, const char *agent, GHashTable *params) { int rc = 0; xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE); xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); crm_xml_add(data, XML_ATTR_ID, id); crm_xml_add(data, "agent", agent); crm_xml_add(data, "namespace", namespace); g_hash_table_foreach(params, hash2field, args); rc = stonith_send_command(stonith, STONITH_OP_DEVICE_ADD, data, NULL, call_options); free_xml(data); return rc; } static int stonith_api_remove_device( stonith_t *stonith, int call_options, const char *name) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, XML_ATTR_ID, name); rc = stonith_send_command(stonith, STONITH_OP_DEVICE_DEL, data, NULL, call_options); free_xml(data); return rc; } static int stonith_api_call( stonith_t *stonith, int call_options, const char *id, const char *action, const char *port, int timeout) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_DEVICE, id); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add(data, F_STONITH_PORT, port); crm_xml_add_int(data, "timeout", timeout); rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, NULL, call_options); free_xml(data); return rc; } static int stonith_api_fence( stonith_t *stonith, int call_options, const char *node, int timeout) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, "target", node); crm_xml_add_int(data, "timeout", timeout); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options); free_xml(data); return rc; } static int stonith_api_unfence( stonith_t *stonith, int call_options, const char *node, int timeout) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, "target", node); crm_xml_add_int(data, "timeout", timeout); rc = stonith_send_command(stonith, STONITH_OP_UNFENCE, data, NULL, call_options); free_xml(data); return rc; } const char * stonith_error2string(enum stonith_errors return_code) { const char *error_msg = NULL; switch(return_code) { case stonith_ok: error_msg = ""; break; case stonith_not_supported: error_msg = ""; break; case stonith_connection: error_msg = ""; break; case stonith_authentication: error_msg = ""; break; case stonith_callback_register: error_msg = ""; break; case stonith_missing: error_msg = ""; break; case stonith_exists: error_msg = ""; break; case stonith_timeout: error_msg = ""; break; case stonith_ipc: error_msg = ""; break; case stonith_peer: error_msg = ""; break; } if(error_msg == NULL) { crm_err("Unknown Stonith error code: %d", return_code); error_msg = ""; } return error_msg; } static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const stonith_notify_client_t *a_client = a; const stonith_notify_client_t *b_client = b; CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if(rc == 0) { if(a_client->callback == b_client->callback) { return 0; } else if(((long)a_client->callback) < ((long)b_client->callback)) { crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->callback, b_client->callback); return -1; } crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->callback, b_client->callback); return 1; } return rc; } static int get_stonith_token(IPC_Channel *ch, char **token) { int rc = stonith_ok; xmlNode *reg_msg = NULL; const char *msg_type = NULL; const char *tmp_ticket = NULL; CRM_CHECK(ch != NULL, return stonith_missing); CRM_CHECK(token != NULL, return stonith_missing); crm_debug_4("Waiting for msg on command channel"); reg_msg = xmlfromIPC(ch, MAX_IPC_DELAY); if(ch->ops->get_chan_status(ch) != IPC_CONNECT) { crm_err("No reply message - disconnected"); free_xml(reg_msg); return stonith_connection; } else if(reg_msg == NULL) { crm_err("No reply message - empty"); return stonith_ipc; } msg_type = crm_element_value(reg_msg, F_STONITH_OPERATION); tmp_ticket = crm_element_value(reg_msg, F_STONITH_CLIENTID); if(safe_str_neq(msg_type, CRM_OP_REGISTER) ) { crm_err("Invalid registration message: %s", msg_type); rc = stonith_callback_register; } else if(tmp_ticket == NULL) { crm_err("No registration token provided"); crm_log_xml_warn(reg_msg, "Bad reply") rc = stonith_peer; } else { crm_debug("Obtained registration token: %s", tmp_ticket); *token = crm_strdup(tmp_ticket); } free_xml(reg_msg); return rc; } xmlNode *stonith_create_op( int call_id, const char *token, const char *op, xmlNode *data, int call_options) { int rc = HA_OK; xmlNode *op_msg = create_xml_node(NULL, "stonith_command"); CRM_CHECK(op_msg != NULL, return NULL); CRM_CHECK(token != NULL, return NULL); crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command"); crm_xml_add(op_msg, F_TYPE, T_STONITH); crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token); crm_xml_add(op_msg, F_STONITH_OPERATION, op); crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id); crm_debug_4("Sending call options: %.8lx, %d", (long)call_options, call_options); crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options); if(data != NULL) { add_message_xml(op_msg, F_STONITH_CALLDATA, data); } if (rc != HA_OK) { crm_err("Failed to create STONITH operation message"); crm_log_xml(LOG_ERR, "op", op_msg); free_xml(op_msg); return NULL; } return op_msg; } static void stonith_destroy_op_callback(gpointer data) { stonith_callback_client_t *blob = data; if(blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } crm_free(blob->timer); crm_free(blob); } static int stonith_api_signoff(stonith_t* stonith) { stonith_private_t *native = stonith->private; crm_debug("Signing out of the STONITH Service"); /* close channels */ if (native->command_channel != NULL) { native->command_channel->ops->destroy( native->command_channel); native->command_channel = NULL; } if (native->callback_source != NULL) { G_main_del_IPC_Channel(native->callback_source); native->callback_source = NULL; } if (native->callback_channel != NULL) { #ifdef BUG native->callback_channel->ops->destroy( native->callback_channel); #endif native->callback_channel = NULL; } stonith->state = stonith_disconnected; return stonith_ok; } static int stonith_api_signon( stonith_t* stonith, const char *name, int *async_fd, int *sync_fd) { int rc = stonith_ok; xmlNode *hello = NULL; char *uuid_ticket = NULL; stonith_private_t *native = stonith->private; crm_debug_4("Connecting command channel"); stonith->state = stonith_connected_command; native->command_channel = init_client_ipc_comms_nodispatch(stonith_channel); if(native->command_channel == NULL) { crm_debug("Connection to command channel failed"); rc = stonith_connection; } else if(native->command_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = stonith_authentication; } if(rc == stonith_ok) { rc = get_stonith_token(native->command_channel, &uuid_ticket); if(rc == stonith_ok) { native->token = uuid_ticket; uuid_ticket = NULL; } else { stonith->state = stonith_disconnected; native->command_channel->ops->disconnect(native->command_channel); return rc; } } native->callback_channel = init_client_ipc_comms_nodispatch( stonith_channel_callback); if(native->callback_channel == NULL) { crm_debug("Connection to callback channel failed"); rc = stonith_connection; } else if(native->callback_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = stonith_authentication; } if(rc == stonith_ok) { native->callback_channel->send_queue->max_qlen = 500; rc = get_stonith_token(native->callback_channel, &uuid_ticket); if(rc == stonith_ok) { crm_free(native->token); native->token = uuid_ticket; } } if(rc == stonith_ok) { CRM_CHECK(native->token != NULL, ;); hello = stonith_create_op(0, native->token, CRM_OP_REGISTER, NULL, 0); crm_xml_add(hello, F_STONITH_CLIENTNAME, name); if(send_ipc_message(native->command_channel, hello) == FALSE) { rc = stonith_callback_register; } free_xml(hello); } if(rc == stonith_ok) { gboolean do_mainloop = TRUE; if(async_fd != NULL) { do_mainloop = FALSE; *async_fd = native->callback_channel->ops->get_recv_select_fd(native->callback_channel); } if(sync_fd != NULL) { do_mainloop = FALSE; *sync_fd = native->callback_channel->ops->get_send_select_fd(native->callback_channel); } if(do_mainloop) { crm_debug_4("Connecting callback channel"); native->callback_source = G_main_add_IPC_Channel( G_PRIORITY_HIGH, native->callback_channel, FALSE, stonith_dispatch, stonith, default_ipc_connection_destroy); if(native->callback_source == NULL) { crm_err("Callback source not recorded"); rc = stonith_connection; } else { set_IPC_Channel_dnotify( native->callback_source, stonith_connection_destroy); } } } if(rc == stonith_ok) { #if HAVE_MSGFROMIPC_TIMEOUT stonith->call_timeout = MAX_IPC_DELAY; #endif crm_debug("Connection to STONITH successful"); return stonith_ok; } crm_debug("Connection to STONITH failed: %s", stonith_error2string(rc)); stonith->cmds->disconnect(stonith); return rc; } static int stonith_set_notification(stonith_t* stonith, const char *callback, int enabled) { - xmlNode *notify_msg = create_xml_node(NULL, "stonith-callback"); + xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__); stonith_private_t *native = stonith->private; if(stonith->state != stonith_disconnected) { crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY); - crm_xml_add(notify_msg, F_STONITH_NOTIFY_TYPE, callback); - crm_xml_add_int(notify_msg, F_STONITH_NOTIFY_ACTIVATE, enabled); + if(enabled) { + crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback); + } else { + crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback); + } send_ipc_message(native->callback_channel, notify_msg); } free_xml(notify_msg); return stonith_ok; } static int stonith_api_add_notification( stonith_t *stonith, const char *event, void (*callback)( const char *event, xmlNode *msg)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; crm_debug_2("Adding callback for %s events (%d)", event, g_list_length(stonith->notify_list)); crm_malloc0(new_client, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom( stonith->notify_list, new_client, stonithlib_GCompareFunc); if(list_item != NULL) { crm_warn("Callback already present"); crm_free(new_client); return stonith_exists; } else { stonith->notify_list = g_list_append( stonith->notify_list, new_client); stonith_set_notification(stonith, event, 1); crm_debug_3("Callback added (%d)", g_list_length(stonith->notify_list)); } return stonith_ok; } static int stonith_api_del_notification( stonith_t *stonith, const char *event, void (*callback)( const char *event, xmlNode *msg)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; crm_debug("Removing callback for %s events", event); crm_malloc0(new_client, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom( stonith->notify_list, new_client, stonithlib_GCompareFunc); stonith_set_notification(stonith, event, 0); if(list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; stonith->notify_list = g_list_remove(stonith->notify_list, list_client); crm_free(list_client); crm_debug_3("Removed callback"); } else { crm_debug_3("Callback not present"); } crm_free(new_client); return stonith_ok; } static gboolean stonith_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_debug("Async call %d timed out after %ds", timer->call_id, timer->timeout); stonith_perform_callback(timer->stonith, NULL, timer->call_id, stonith_timeout); /* Always return TRUE, never remove the handler * We do that in stonith_del_callback() */ return TRUE; } static int stonith_api_add_callback( stonith_t *stonith, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback)(const xmlNode*, int, int, xmlNode*,void*)) { stonith_callback_client_t *blob = NULL; CRM_CHECK(stonith != NULL, return stonith_missing); CRM_CHECK(stonith->private != NULL, return stonith_missing); if(call_id == 0) { stonith_private_t *private = stonith->private; private->op_callback = callback; } else if(call_id < 0) { if(only_success == FALSE) { callback(NULL, call_id, call_id, NULL, user_data); } else { crm_warn("STONITH call failed: %s", stonith_error2string(call_id)); } return FALSE; } crm_malloc0(blob, sizeof(stonith_callback_client_t)); blob->id = callback_name; blob->only_success = only_success; blob->user_data = user_data; blob->callback = callback; if(timeout > 0) { struct timer_rec_s *async_timer = NULL; crm_malloc0(async_timer, sizeof(struct timer_rec_s)); blob->timer = async_timer; async_timer->stonith = stonith; async_timer->call_id = call_id; async_timer->timeout = timeout*1000; async_timer->ref = g_timeout_add( async_timer->timeout, stonith_async_timeout_handler, async_timer); } g_hash_table_insert(stonith_op_callback_table, GINT_TO_POINTER(call_id), blob); return TRUE; } static int stonith_api_del_callback(int call_id, gboolean all_callbacks) { if(all_callbacks) { if(stonith_op_callback_table != NULL) { g_hash_table_destroy(stonith_op_callback_table); } stonith_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); } else { g_hash_table_remove(stonith_op_callback_table, GINT_TO_POINTER(call_id)); } return stonith_ok; } static void stonith_dump_pending_op( gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); stonith_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, crm_str(blob->id)); } void stonith_dump_pending_callbacks(void) { if(stonith_op_callback_table == NULL) { return; } return g_hash_table_foreach( stonith_op_callback_table, stonith_dump_pending_op, NULL); } void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc) { xmlNode *output = NULL; stonith_private_t *private = NULL; stonith_callback_client_t *blob = NULL; stonith_callback_client_t local_blob; CRM_CHECK(stonith != NULL, return); CRM_CHECK(stonith->private != NULL, return); private = stonith->private; local_blob.id = NULL; local_blob.callback = NULL; local_blob.user_data = NULL; local_blob.only_success = FALSE; if(msg != NULL) { crm_element_value_int(msg, F_STONITH_RC, &rc); crm_element_value_int(msg, F_STONITH_CALLID, &call_id); output = get_message_xml(msg, F_STONITH_CALLDATA); } blob = g_hash_table_lookup( stonith_op_callback_table, GINT_TO_POINTER(call_id)); if(blob != NULL) { local_blob = *blob; blob = NULL; stonith_api_del_callback(call_id, FALSE); } else { crm_debug_2("No callback found for call %d", call_id); local_blob.callback = NULL; } if(stonith == NULL) { crm_debug("No stonith object supplied"); } if(local_blob.callback != NULL && (rc == stonith_ok || local_blob.only_success == FALSE)) { crm_debug_2("Invoking callback %s for call %d", crm_str(local_blob.id), call_id); local_blob.callback(msg, call_id, rc, output, local_blob.user_data); } else if(private->op_callback == NULL && rc != stonith_ok) { crm_warn("STONITH command failed: %s", stonith_error2string(rc)); crm_log_xml(LOG_DEBUG, "Failed STONITH Update", msg); } if(private->op_callback != NULL) { crm_debug_2("Invoking global callback for call %d", call_id); private->op_callback(msg, call_id, rc, output, stonith); } crm_debug_4("OP callback activated."); } static void stonith_send_notification(gpointer data, gpointer user_data) { xmlNode *msg = user_data; stonith_notify_client_t *entry = data; const char *event = NULL; if(msg == NULL) { crm_warn("Skipping callback - NULL message"); return; } event = crm_element_value(msg, F_SUBTYPE); if(entry == NULL) { crm_warn("Skipping callback - NULL callback client"); return; } else if(entry->callback == NULL) { crm_warn("Skipping callback - NULL callback"); return; } else if(safe_str_neq(entry->event, event)) { crm_debug_4("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } crm_debug_4("Invoking callback for %p/%s event...", entry, event); entry->callback(event, msg); crm_debug_4("Callback invoked..."); } static gboolean timer_expired = FALSE; int stonith_send_command( stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options) { int rc = HA_OK; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; stonith_private_t *native = stonith->private; if(stonith->state == stonith_disconnected) { return stonith_connection; } if(output_data != NULL) { *output_data = NULL; } if(op == NULL) { crm_err("No operation specified"); return stonith_missing; } stonith->call_id++; /* prevent call_id from being negative (or zero) and conflicting * with the stonith_errors enum * use 2 because we use it as (stonith->call_id - 1) below */ if(stonith->call_id < 1) { stonith->call_id = 1; } CRM_CHECK(native->token != NULL, ;); op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options); if(op_msg == NULL) { return stonith_missing; } crm_debug_3("Sending %s message to STONITH service", op); if(send_ipc_message(native->command_channel, op_msg) == FALSE) { crm_err("Sending message to STONITH service FAILED"); free_xml(op_msg); return stonith_ipc; } else { crm_debug_3("Message sent"); } free_xml(op_msg); if((call_options & stonith_discard_reply)) { crm_debug_3("Discarding reply"); return stonith_ok; } else if(!(call_options & stonith_sync_call)) { crm_debug_3("Async call, returning"); CRM_CHECK(stonith->call_id != 0, return stonith_ipc); return stonith->call_id; } rc = IPC_OK; crm_debug_3("Waiting for a syncronous reply"); #ifndef HAVE_MSGFROMIPC_TIMEOUT sync_timer.ref = 0; if(stonith->call_timeout > 0) { timer_expired = FALSE; sync_timer.call_id = stonith->call_id; sync_timer.timeout = stonith->call_timeout*1000; sync_timer.ref = g_timeout_add( sync_timer.timeout, stonith_timeout_handler, &sync_timer); } #endif rc = stonith_ok; while(timer_expired == FALSE && IPC_ISRCONN(native->command_channel)) { int reply_id = -1; int msg_id = stonith->call_id; op_reply = xmlfromIPC(native->command_channel, stonith->call_timeout); if(op_reply == NULL) { rc = stonith_peer; break; } crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id); if(reply_id <= 0) { rc = stonith_peer; break; } else if(reply_id == msg_id) { crm_debug_3("Syncronous reply received"); crm_log_xml(LOG_MSG, "Reply", op_reply); if(crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) { rc = stonith_peer; } if(output_data != NULL && is_not_set(call_options, stonith_discard_reply)) { xmlNode *tmp = get_message_xml(op_reply, F_STONITH_CALLDATA); if(tmp != NULL) { *output_data = copy_xml(tmp); } } break; } else if(reply_id < msg_id) { crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else if((reply_id - 10000) > msg_id) { /* wrap-around case */ crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else { crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id); } free_xml(op_reply); op_reply = NULL; } if(IPC_ISRCONN(native->command_channel) == FALSE) { crm_err("STONITH disconnected: %d", native->command_channel->ch_status); stonith->state = stonith_disconnected; } if(op_reply == NULL && stonith->state == stonith_disconnected) { rc = stonith_connection; } else if(rc == stonith_ok && op_reply == NULL) { rc = stonith_peer; } #ifndef HAVE_MSGFROMIPC_TIMEOUT if(sync_timer.ref > 0) { g_source_remove(sync_timer.ref); sync_timer.ref = 0; } #endif free_xml(op_reply); return rc; } static gboolean stonith_msgready(stonith_t* stonith) { stonith_private_t *private = NULL; if (stonith == NULL) { crm_err("No STONITH!"); return FALSE; } private = stonith->private; if(private->command_channel != NULL) { /* drain the channel */ IPC_Channel *cmd_ch = private->command_channel; xmlNode *cmd_msg = NULL; while(cmd_ch->ch_status != IPC_DISCONNECT && cmd_ch->ops->is_message_pending(cmd_ch)) { /* this will happen when the STONITH exited from beneath us */ cmd_msg = xmlfromIPC(cmd_ch, MAX_IPC_DELAY); free_xml(cmd_msg); } } else { crm_err("No command channel"); } if(private->callback_channel == NULL) { crm_err("No callback channel"); return FALSE; } else if(private->callback_channel->ch_status == IPC_DISCONNECT) { crm_info("Lost connection to the STONITH service [%d].", private->callback_channel->farside_pid); return FALSE; } else if(private->callback_channel->ops->is_message_pending( private->callback_channel)) { crm_debug_4("Message pending on command channel [%d]", private->callback_channel->farside_pid); return TRUE; } crm_debug_3("No message pending"); return FALSE; } static int stonith_rcvmsg(stonith_t* stonith) { const char *type = NULL; xmlNode* msg = NULL; stonith_private_t *private = NULL; if (stonith == NULL) { crm_err("No STONITH!"); return FALSE; } private = stonith->private; /* if it is not blocking mode and no message in the channel, return */ if (stonith_msgready(stonith) == FALSE) { crm_debug_3("No message ready and non-blocking..."); return 0; } /* IPC_INTR is not a factor here */ msg = xmlfromIPC(private->callback_channel, MAX_IPC_DELAY); if (msg == NULL) { crm_warn("Received a NULL msg from STONITH service."); return 0; } /* do callbacks */ type = crm_element_value(msg, F_TYPE); crm_debug_4("Activating %s callbacks...", type); if(safe_str_eq(type, T_STONITH_NG)) { stonith_perform_callback(stonith, msg, 0, 0); } else if(safe_str_eq(type, T_STONITH_NOTIFY)) { g_list_foreach(stonith->notify_list, stonith_send_notification, msg); } else { crm_err("Unknown message type: %s", type); } free_xml(msg); return 1; } gboolean stonith_dispatch(IPC_Channel *channel, gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *private = NULL; gboolean stay_connected = TRUE; CRM_CHECK(stonith != NULL, return FALSE); private = stonith->private; CRM_CHECK(private->callback_channel == channel, return FALSE); while(stonith_msgready(stonith)) { /* invoke the callbacks but dont block */ int rc = stonith_rcvmsg(stonith); if( rc < 0) { crm_err("Message acquisition failed: %d", rc); break; } else if(rc == 0) { break; } } if(private->callback_channel && private->callback_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the STONITH service [%d/callback].", channel->farside_pid); private->callback_source = NULL; stay_connected = FALSE; } if(private->command_channel && private->command_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the STONITH service [%d/command].", channel->farside_pid); private->callback_source = NULL; stay_connected = FALSE; } return stay_connected; } static int stonith_api_free (stonith_t* stonith) { int rc = stonith_ok; if(stonith->state != stonith_disconnected) { rc = stonith->cmds->disconnect(stonith); } if(stonith->state == stonith_disconnected) { stonith_private_t *private = stonith->private; crm_free(private->token); crm_free(stonith->private); crm_free(stonith->cmds); crm_free(stonith); } return rc; } void stonith_api_delete(stonith_t *stonith) { GList *list = stonith->notify_list; while(list != NULL) { stonith_notify_client_t *client = g_list_nth_data(list, 0); list = g_list_remove(list, client); crm_free(client); } g_hash_table_destroy(stonith_op_callback_table); stonith->cmds->free(stonith); stonith = NULL; } stonith_t *stonith_api_new(void) { stonith_t* new_stonith = NULL; stonith_private_t* private = NULL; crm_malloc0(new_stonith, sizeof(stonith_t)); crm_malloc0(private, sizeof(stonith_private_t)); new_stonith->private = private; if(stonith_op_callback_table != NULL) { g_hash_table_destroy(stonith_op_callback_table); stonith_op_callback_table = NULL; } if(stonith_op_callback_table == NULL) { stonith_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); } new_stonith->call_id = 1; new_stonith->notify_list = NULL; new_stonith->state = stonith_disconnected; crm_malloc0(new_stonith->cmds, sizeof(stonith_api_operations_t)); new_stonith->cmds->free = stonith_api_free; new_stonith->cmds->connect = stonith_api_signon; new_stonith->cmds->disconnect = stonith_api_signoff; new_stonith->cmds->call = stonith_api_call; new_stonith->cmds->fence = stonith_api_fence; new_stonith->cmds->unfence = stonith_api_unfence; new_stonith->cmds->remove_device = stonith_api_remove_device; new_stonith->cmds->register_device = stonith_api_register_device; new_stonith->cmds->remove_callback = stonith_api_del_callback; new_stonith->cmds->register_callback = stonith_api_add_callback; new_stonith->cmds->remove_notification = stonith_api_del_notification; new_stonith->cmds->register_notification = stonith_api_add_notification; return new_stonith; }