diff --git a/fencing/admin.c b/fencing/admin.c index 59b13411a7..da7638a904 100644 --- a/fencing/admin.c +++ b/fencing/admin.c @@ -1,343 +1,347 @@ /* * Copyright (C) 2009 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct crm_option long_options[] = { {"help", 0, 0, '?', "\tThis text"}, {"version", 0, 0, '$', "\tVersion information" }, {"verbose", 0, 0, 'V', "\tIncrease debug output"}, {"quiet", 0, 0, 'q', "\tPrint only essential output"}, {"list", 1, 0, 'l', "List devices that can terminate the specified host"}, {"list-registered", 0, 0, 'L', "List all registered devices"}, {"list-installed", 0, 0, 'I', "List all installed devices"}, {"metadata", 0, 0, 'M', "Check the device's metadata"}, {"query", 1, 0, 'Q', "Check the device's status"}, {"fence", 1, 0, 'F', "Fence the named host"}, {"unfence", 1, 0, 'U', "Unfence the named host"}, {"reboot", 1, 0, 'B', "Reboot the named host"}, {"confirm", 1, 0, 'C', "Confirm the named host is now safely down"}, {"history", 1, 0, 'H', "Retrieve last fencing operation"}, {"register", 1, 0, 'R', "Register a stonith device"}, {"deregister", 1, 0, 'D', "De-register a stonith device"}, {"env-option", 1, 0, 'e'}, {"option", 1, 0, 'o'}, {"agent", 1, 0, 'a'}, {"list-all", 0, 0, 'L', "legacy alias for --list-registered"}, {0, 0, 0, 0} }; int st_opts = st_opt_sync_call; int main(int argc, char ** argv) { int flag; int rc = 0; int quiet = 0; int verbose = 0; int argerr = 0; int option_index = 0; char name[512]; char value[512]; const char *agent = NULL; const char *device = NULL; const char *target = NULL; char action = 0; stonith_t *st = NULL; stonith_key_value_t *params = NULL; stonith_key_value_t *devices = NULL; stonith_key_value_t *dIter = NULL; crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv); crm_set_options(NULL, "mode [options]", long_options, "Provides access to the stonith-ng API.\n"); while (1) { flag = crm_get_option(argc, argv, &option_index); if (flag == -1) break; switch(flag) { case 'V': verbose = 1; alter_debug(DEBUG_INC); cl_log_enable_stderr(1); break; case '$': case '?': crm_help(flag, LSB_EXIT_OK); break; case 'L': case 'I': action = flag; break; case 'q': quiet = 1; break; case 'Q': case 'R': case 'D': action = flag; device = optarg; break; case 'a': agent = optarg; break; case 'l': target = optarg; action = 'L'; break; case 'M': action = flag; break; case 'B': case 'F': case 'U': case 'C': case 'H': cl_log_enable_stderr(1); target = optarg; action = flag; break; case 'o': crm_info("Scanning: -o %s", optarg); rc = sscanf(optarg, "%[^=]=%[^=]", name, value); if(rc != 2) { crm_err("Invalid option: -o %s", optarg); ++argerr; } else { crm_info("Got: '%s'='%s'", name, value); stonith_key_value_add(params, name, value); } break; case 'e': { char *key = crm_concat("OCF_RESKEY", optarg, '_'); const char *env = getenv(key); if(env == NULL) { crm_err("Invalid option: -e %s", optarg); ++argerr; } else { crm_info("Got: '%s'='%s'", optarg, env); stonith_key_value_add( params, optarg, env); } } break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { crm_help('?', LSB_EXIT_GENERIC); } crm_debug("Create"); st = stonith_api_new(); if(action != 'M' && action != 'I') { rc = st->cmds->connect(st, crm_system_name, NULL); crm_debug("Connect: %d", rc); if(rc < 0) { goto done; } } switch(action) { case 'I': rc = st->cmds->list(st, st_opt_sync_call, NULL, &devices, 0); for(dIter = devices; dIter; dIter = dIter->next ) { fprintf( stdout, " %s\n", dIter->value ); } if(rc == 0) { fprintf(stderr, "No devices found\n"); } else if(rc > 0) { fprintf(stderr, "%d devices found\n", rc); rc = 0; } stonith_key_value_freeall(devices, 1, 1); break; case 'L': rc = st->cmds->query(st, st_opts, target, &devices, 10); for(dIter = devices; dIter; dIter = dIter->next ) { fprintf( stdout, " %s\n", dIter->value ); crm_free(dIter->value); } if(rc == 0) { fprintf(stderr, "No devices found\n"); } else if(rc > 0) { fprintf(stderr, "%d devices found\n", rc); rc = 0; } stonith_key_value_freeall(devices, 1, 1); break; case 'Q': rc = st->cmds->call(st, st_opts, device, "monitor", NULL, 10); if(rc < 0) { rc = st->cmds->call(st, st_opts, device, "list", NULL, 10); } break; case 'R': rc = st->cmds->register_device(st, st_opts, device, "stonith-ng", agent, params); break; case 'D': rc = st->cmds->remove_device(st, st_opts, device); break; case 'M': - { + if (agent == NULL) { + printf("Please specify an agent to query using -a,--agent [value]\n"); + return -1; + + } else { char *buffer = NULL; st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, 0); printf("%s\n", buffer); crm_free(buffer); } break; case 'C': rc = st->cmds->confirm(st, st_opts, target); break; case 'B': rc = st->cmds->fence(st, st_opts, target, "reboot", 120); break; case 'F': rc = st->cmds->fence(st, st_opts, target, "off", 120); break; case 'U': rc = st->cmds->fence(st, st_opts, target, "on", 120); break; case 'H': { stonith_history_t *history, *hp, *latest = NULL; rc = st->cmds->history(st, st_opts, target, &history, 120); for(hp = history; hp; hp = hp->next) { char *action_s = NULL; time_t complete = hp->completed; if(hp->state == st_done) { latest = hp; } if(quiet || !verbose) { continue; } else if(hp->action == NULL) { action_s = crm_strdup("unknown"); } else if(hp->action[0] != 'r') { action_s = crm_concat("turn", hp->action, ' '); } else { action_s = crm_strdup(hp->action); } if(hp->state == st_failed) { printf("%s failed to %s node %s on behalf of %s at %s\n", hp->delegate?hp->delegate:"We", action_s, hp->target, hp->origin, ctime(&complete)); } else if(hp->state == st_done) { printf("%s was able to %s node %s on behalf of %s at %s\n", hp->delegate?hp->delegate:"We", action_s, hp->target, hp->origin, ctime(&complete)); } else { printf("%s wishes to %s node %s - %d %d\n", hp->origin, action_s, hp->target, hp->state, hp->completed); } crm_free(action_s); } if(latest) { if(quiet) { printf("%d\n", latest->completed); } else { char *action_s = NULL; time_t complete = latest->completed; if(latest->action == NULL) { action_s = crm_strdup("unknown"); } else if(latest->action[0] != 'r') { action_s = crm_concat("turn", latest->action, ' '); } else { action_s = crm_strdup(latest->action); } printf("%s was able to %s node %s on behalf of %s at %s\n", latest->delegate?latest->delegate:"We", action_s, latest->target, latest->origin, ctime(&complete)); crm_free(action_s); } } } break; } done: if(rc < 0) { printf("Command failed: %s\n", stonith_error2string(rc)); } stonith_key_value_freeall(params, 1, 1); st->cmds->disconnect(st); crm_debug("Disconnect: %d", rc); crm_debug("Destroy"); stonith_api_delete(st); return rc; } diff --git a/fencing/commands.c b/fencing/commands.c index 8267ce93db..1419899b79 100644 --- a/fencing/commands.c +++ b/fencing/commands.c @@ -1,954 +1,1006 @@ /* * Copyright (C) 2009 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include GHashTable *device_list = NULL; static int active_children = 0; +static gboolean stonith_device_dispatch(gpointer user_data); static void exec_child_done(ProcTrack* proc, int status, int signo, int rc, int waslogged); static void exec_child_new(ProcTrack* p) { active_children++; } static const char *exec_child_name(ProcTrack* p) { async_command_t *cmd = proctrack_data(p); return cmd->client?cmd->client:cmd->remote; } static ProcTrack_ops StonithdProcessTrackOps = { exec_child_done, exec_child_new, exec_child_name, }; static void free_async_command(async_command_t *cmd) { + crm_free(cmd->device); crm_free(cmd->action); crm_free(cmd->victim); crm_free(cmd->remote); crm_free(cmd->client); crm_free(cmd->origin); crm_free(cmd->op); crm_free(cmd); } static async_command_t *create_async_command(xmlNode *msg, const char *action) { async_command_t *cmd = NULL; CRM_CHECK(action != NULL, crm_log_xml_warn(msg, "NoAction"); return NULL); crm_malloc0(cmd, sizeof(async_command_t)); crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id)); crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options)); crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->timeout)); cmd->origin = crm_element_value_copy(msg, F_ORIG); cmd->remote = crm_element_value_copy(msg, F_STONITH_REMOTE); cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID); cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION); cmd->action = crm_strdup(action); cmd->victim = crm_element_value_copy(msg, F_STONITH_TARGET); cmd->pt_ops = &StonithdProcessTrackOps; CRM_CHECK(cmd->op != NULL, crm_log_xml_warn(msg, "NoOp"); free_async_command(cmd); return NULL); CRM_CHECK(cmd->client != NULL || cmd->remote != NULL, crm_log_xml_warn(msg, "NoClient")); - return cmd; } +static gboolean stonith_device_execute(stonith_device_t *device) +{ + int rc = 0; + int exec_rc = 0; + async_command_t *cmd = NULL; + CRM_CHECK(device != NULL, return FALSE); + + if(device->active_pid) { + crm_trace("%s is still active with pid %u", device->id, device->active_pid); + return TRUE; + } + + if(device->pending_ops) { + GList *first = device->pending_ops; + device->pending_ops = g_list_remove_link(device->pending_ops, first); + cmd = first->data; + g_list_free_1(first); + } + + if(cmd == NULL) { + crm_info("Nothing to do for %s", device->id); + return TRUE; + } + + cmd->device = crm_strdup(device->id); + exec_rc = run_stonith_agent(device->agent, cmd->action, cmd->victim, + device->params, device->aliases, &rc, NULL, cmd); + + if(exec_rc > 0) { + crm_debug("Operation %s%s%s on %s is active with pid: %d", + cmd->action, cmd->victim?" for node ":"", cmd->victim?cmd->victim:"", + device->id, exec_rc); + device->active_pid = exec_rc; + + } else { + struct _ProcTrack p; + memset(&p, 0, sizeof(struct _ProcTrack)); + p.privatedata = cmd; + + crm_warn("Operation %s%s%s on %s failed (%d/%d)", + cmd->action, cmd->victim?" for node ":"", cmd->victim?cmd->victim:"", + device->id, exec_rc, rc); + exec_child_done(&p, 0, 0, rc<0?rc:exec_rc, 0); + } + return TRUE; +} + +static gboolean stonith_device_dispatch(gpointer user_data) +{ + return stonith_device_execute(user_data); +} + +static void schedule_stonith_command(async_command_t *cmd, stonith_device_t *device) +{ + CRM_CHECK(cmd != NULL, return); + CRM_CHECK(device != NULL, return); + + crm_trace("Scheduling %s on %s", cmd->action, device->id); + device->pending_ops = g_list_append(device->pending_ops, cmd); + mainloop_set_trigger(device->work); +} + static void free_device(gpointer data) { + GListPtr gIter = NULL; stonith_device_t *device = data; g_hash_table_destroy(device->params); g_hash_table_destroy(device->aliases); + + for(gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) { + struct _ProcTrack p; + async_command_t *cmd = gIter->data; + + memset(&p, 0, sizeof(struct _ProcTrack)); + p.privatedata = cmd; + + crm_warn("Removal of device '%s' purged operation %s", device->id, cmd->action); + exec_child_done(&p, 0, 0, st_err_unknown_device, 0); + free_async_command(cmd); + } + g_list_free(device->pending_ops); + slist_basic_destroy(device->targets); crm_free(device->namespace); crm_free(device->agent); crm_free(device->id); crm_free(device); } static GHashTable *build_port_aliases(const char *hostmap, GListPtr *targets) { char *name = NULL; int last = 0, lpc = 0, max = 0, added = 0; GHashTable *aliases = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if(hostmap == NULL) { return aliases; } max = strlen(hostmap); for(; lpc <= max; lpc++) { switch(hostmap[lpc]) { /* Assignment chars */ case '=': case ':': if(lpc > last) { crm_free(name); crm_malloc0(name, 1 + lpc - last); strncpy(name, hostmap + last, lpc - last); } last = lpc + 1; break; /* Delimeter chars */ /* case ',': Potentially used to specify multiple ports */ case 0: case ';': case ' ': case '\t': if(name) { char *value = NULL; crm_malloc0(value, 1 + lpc - last); strncpy(value, hostmap + last, lpc - last); crm_debug("Adding alias '%s'='%s'", name, value); g_hash_table_replace(aliases, name, value); if(targets) { *targets = g_list_append(*targets, crm_strdup(value)); } value=NULL; name=NULL; added++; } else if(lpc > last) { crm_debug("Parse error at offset %d near '%s'", lpc-last, hostmap+last); } last = lpc + 1; break; } if(hostmap[lpc] == 0) { break; } } if(added == 0) { crm_info("No host mappings detected in '%s'", hostmap); } crm_free(name); return aliases; } static void parse_host_line(const char *line, GListPtr *output) { int lpc = 0; int max = 0; int last = 0; if(line) { max = strlen(line); } else { return; } /* Check for any complaints about additional parameters that the device doesn't understand */ if(strstr(line, "invalid") || strstr(line, "variable")) { crm_debug("Skipping: %s", line); return; } crm_debug_2("Processing: %s", line); /* Skip initial whitespace */ for(lpc = 0; lpc <= max && isspace(line[lpc]); lpc++) { last = lpc+1; } /* Now the actual content */ for(lpc = 0; lpc <= max; lpc++) { gboolean a_space = isspace(line[lpc]); if(a_space && lpc < max && isspace(line[lpc+1])) { /* fast-forward to the end of the spaces */ } else if(a_space || line[lpc] == ',' || line[lpc] == 0) { int rc = 0; char *entry = NULL; crm_malloc0(entry, 1 + lpc - last); rc = sscanf(line+last, "%[a-zA-Z0-9_-.]", entry); if(rc != 1) { crm_warn("Could not parse (%d %d): %s", last, lpc, line+last); } else if(safe_str_neq(entry, "on") && safe_str_neq(entry, "off")) { crm_debug_2("Adding '%s'", entry); *output = g_list_append(*output, entry); entry = NULL; } crm_free(entry); last = lpc + 1; } } } static GListPtr parse_host_list(const char *hosts) { int lpc = 0; int max = 0; int last = 0; GListPtr output = NULL; if(hosts == NULL) { return output; } max = strlen(hosts); for(lpc = 0; lpc <= max; lpc++) { if(hosts[lpc] == '\n' || hosts[lpc] == 0) { char *line = NULL; crm_malloc0(line, 2 + lpc - last); snprintf(line, 1 + lpc - last, "%s", hosts+last); parse_host_line(line, &output); crm_free(line); last = lpc + 1; } } return output; } static stonith_device_t *build_device_from_xml(xmlNode *msg) { xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR); stonith_device_t *device = NULL; crm_malloc0(device, sizeof(stonith_device_t)); device->id = crm_element_value_copy(dev, XML_ATTR_ID); device->agent = crm_element_value_copy(dev, "agent"); device->namespace = crm_element_value_copy(dev, "namespace"); device->params = xml2list(dev); + device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device); /* TODO: Hook up priority */ return device; } static int stonith_device_register(xmlNode *msg) { const char *value = NULL; stonith_device_t *device = build_device_from_xml(msg); value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTLIST); if(value) { device->targets = parse_host_list(value); } value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTMAP); device->aliases = build_port_aliases(value, &(device->targets)); g_hash_table_replace(device_list, device->id, device); crm_info("Added '%s' to the device list (%d active devices)", device->id, g_hash_table_size(device_list)); return stonith_ok; } static int stonith_device_remove(xmlNode *msg) { xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR); const char *id = crm_element_value(dev, XML_ATTR_ID); if(g_hash_table_remove(device_list, id)) { crm_info("Removed '%s' from the device list (%d active devices)", id, g_hash_table_size(device_list)); } else { crm_info("Device '%s' not found (%d active devices)", id, g_hash_table_size(device_list)); } return stonith_ok; } static gboolean string_in_list(GListPtr list, const char *item) { int lpc = 0; int max = g_list_length(list); for(lpc = 0; lpc < max; lpc ++) { const char *value = g_list_nth_data(list, lpc); if(safe_str_eq(item, value)) { return TRUE; } } return FALSE; } static int stonith_device_action(xmlNode *msg, char **output) { int rc = stonith_ok; xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR); const char *id = crm_element_value(dev, F_STONITH_DEVICE); const char *action = crm_element_value(dev, F_STONITH_ACTION); async_command_t *cmd = NULL; stonith_device_t *device = NULL; if(id) { crm_debug_2("Looking for '%s'", id); device = g_hash_table_lookup(device_list, id); - - } else { - CRM_CHECK(safe_str_eq(action, "metadata"), crm_log_xml_warn(msg, "StrangeOp")); - - device = build_device_from_xml(msg); - if(device != NULL && device->id == NULL) { - device->id = crm_strdup(device->agent); - } } if(device) { - int exec_rc = 0; - const char *victim = NULL; - cmd = create_async_command(msg, action); if(cmd == NULL) { free_device(device); return st_err_internal; } - cmd->device = crm_strdup(device->id); - crm_debug("Calling '%s' with action '%s'%s%s", - device->id, action, victim?" on port ":"", victim?victim:""); - - exec_rc = run_stonith_agent( - device->agent, action, cmd->victim, device->params, device->aliases, &rc, output, cmd); - if(exec_rc < 0 || rc != 0) { - crm_warn("Operation %s on %s failed (%d/%d): %.100s", - action, device->id, exec_rc, rc, *output); - - } else if(exec_rc > 0) { - crm_debug("Operation %s on %s active with pid: %d", action, device->id, exec_rc); - rc = exec_rc; - - } else { - crm_info("Operation %s on %s passed: %.100s", action, device->id, *output); - } + schedule_stonith_command(cmd, device); + rc = stonith_pending; } else { - crm_notice("Device %s not found", id); + crm_notice("Device %s not found", id?id:""); rc = st_err_unknown_device; } - - if(id == NULL) { - free_device(device); - } return rc; } static gboolean can_fence_host_with_device(stonith_device_t *dev, const char *host) { gboolean can = FALSE; const char *alias = host; const char *check_type = NULL; if(dev == NULL) { return FALSE; } else if(host == NULL) { return TRUE; } if(g_hash_table_lookup(dev->aliases, host)) { alias = g_hash_table_lookup(dev->aliases, host); } check_type = g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTCHECK); if(check_type == NULL) { if(g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTLIST)) { check_type = "static-list"; } else { check_type = "dynamic-list"; } } if(safe_str_eq(check_type, "none")) { can = TRUE; } else if(safe_str_eq(check_type, "static-list")) { /* Presence in the hostmap is sufficient * Only use if all hosts on which the device can be active can always fence all listed hosts */ if(string_in_list(dev->targets, host)) { can = TRUE; } } else if(safe_str_eq(check_type, "dynamic-list")) { time_t now = time(NULL); /* Host/alias must be in the list output to be eligable to be fenced * * Will cause problems if down'd nodes aren't listed or (for virtual nodes) * if the guest is still listed despite being moved to another machine */ if(dev->targets_age < 0) { crm_trace("Port list queries disabled for %s", dev->id); } else if(dev->targets == NULL || dev->targets_age + 60 < now) { char *output = NULL; int rc = stonith_ok; int exec_rc = stonith_ok; /* Check for the target's presence in the output of the 'list' command */ slist_basic_destroy(dev->targets); dev->targets = NULL; exec_rc = run_stonith_agent(dev->agent, "list", NULL, dev->params, NULL, &rc, &output, NULL); if(exec_rc < 0 || rc != 0) { crm_notice("Disabling port list queries for %s (%d/%d): %s", dev->id, exec_rc, rc, output); dev->targets_age = -1; } else { crm_info("Refreshing port list for %s", dev->id); dev->targets = parse_host_list(output); dev->targets_age = now; } crm_free(output); } if(string_in_list(dev->targets, alias)) { can = TRUE; } } else if(safe_str_eq(check_type, "status")) { int rc = 0; int exec_rc = 0; /* Run the status operation for the device/target combination * Will cause problems if the device doesn't return 2 for down'd nodes or * (for virtual nodes) if the device doesn't return 1 for guests that * have been moved to another host */ exec_rc = run_stonith_agent( dev->agent, "status", host, dev->params, dev->aliases, &rc, NULL, NULL); if(exec_rc != 0) { crm_err("Could not invoke %s: rc=%d", dev->id, exec_rc); } else if(rc == 1 /* unkown */) { crm_debug_2("Host %s is not known by %s", host, dev->id); } else if(rc == 0 /* active */ || rc == 2 /* inactive */) { can = TRUE; } else { crm_err("Unkown result calling %s for %s with %s: rc=%d", "status", host, dev->id, rc); } } else { crm_err("Unknown check type: %s", check_type); } if(safe_str_eq(host, alias)) { crm_info("%s can%s fence %s: %s", dev->id, can?"":" not", host, check_type); } else { crm_info("%s can%s fence %s (aka. '%s'): %s", dev->id, can?"":" not", host, alias, check_type); } return can; } struct device_search_s { const char *host; GListPtr capable; }; static void search_devices( gpointer key, gpointer value, gpointer user_data) { stonith_device_t *dev = value; struct device_search_s *search = user_data; if(can_fence_host_with_device(dev, search->host)) { search->capable = g_list_append(search->capable, value); } } static int stonith_query(xmlNode *msg, xmlNode **list) { struct device_search_s search; int available_devices = 0; xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_DEBUG_3); search.host = NULL; search.capable = NULL; if(dev) { search.host = crm_element_value(dev, F_STONITH_TARGET); } crm_log_xml_debug(msg, "Query"); g_hash_table_foreach(device_list, search_devices, &search); available_devices = g_list_length(search.capable); if(search.host) { crm_debug("Found %d matching devices for '%s'", available_devices, search.host); } else { crm_debug("%d devices installed", available_devices); } /* Pack the results into data */ if(list) { GListPtr lpc = NULL; *list = create_xml_node(NULL, __FUNCTION__); crm_xml_add(*list, F_STONITH_TARGET, search.host); crm_xml_add_int(*list, "st-available-devices", available_devices); for(lpc = search.capable; lpc != NULL; lpc = lpc->next) { stonith_device_t *device = (stonith_device_t*)lpc->data; dev = create_xml_node(*list, F_STONITH_DEVICE); crm_xml_add(dev, XML_ATTR_ID, device->id); crm_xml_add(dev, "namespace", device->namespace); crm_xml_add(dev, "agent", device->agent); if(search.host == NULL) { xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS); g_hash_table_foreach(device->params, hash2field, attrs); } } } g_list_free(search.capable); return available_devices; } static void log_operation(async_command_t *cmd, int rc, int pid, const char *next, const char *output) { if(rc == 0) { next = NULL; } if(cmd->victim != NULL) { do_crm_log(rc==0?LOG_INFO:LOG_ERR, "Operation '%s' [%d] (call %d from %s) for host '%s' with device '%s' returned: %d%s%s", cmd->action, pid, cmd->id, cmd->client, cmd->victim, cmd->device, rc, next?". Trying: ":"", next?next:""); } else { do_crm_log(rc==0?LOG_DEBUG:LOG_NOTICE, "Operation '%s' [%d] for device '%s' returned: %d%s%s", cmd->action, pid, cmd->device, rc, next?". Trying: ":"", next?next:""); } if(output) { /* Logging the whole string confuses syslog when the string is xml */ char *local_copy = crm_strdup(output); int lpc = 0, last = 0, more = strlen(local_copy); for(lpc = 0; lpc < more; lpc++) { if(local_copy[lpc] == '\n' || local_copy[lpc] == 0) { local_copy[lpc] = 0; do_crm_log(rc==0?LOG_INFO:LOG_ERR, "%s: %s", cmd->device, local_copy+last); last = lpc+1; } } crm_debug("%s output: %s (total %d bytes)", cmd->device, local_copy+last, more); crm_free(local_copy); } } #define READ_MAX 500 static void exec_child_done(ProcTrack* proc, int status, int signum, int rc, int waslogged) { int len = 0; int more = 0; gboolean bcast = FALSE; char *output = NULL; xmlNode *data = NULL; xmlNode *reply = NULL; int pid = proctrack_pid(proc); + stonith_device_t *device = NULL; async_command_t *cmd = proctrack_data(proc); CRM_CHECK(cmd != NULL, return); active_children--; + /* The device is ready to do something else now */ + device = g_hash_table_lookup(device_list, cmd->device); + if(device) { + device->active_pid = 0; + mainloop_set_trigger(device->work); + } + if( signum ) { rc = st_err_signal; if( proctrack_timedout(proc) ) { crm_warn("Child '%d' performing action '%s' with '%s' timed out", pid, cmd->action, cmd->device); rc = st_err_timeout; } } do { char buffer[READ_MAX]; errno = 0; - memset(&buffer, 0, READ_MAX); - more = read(cmd->stdout, buffer, READ_MAX-1); - do_crm_log(status!=0?LOG_DEBUG:LOG_DEBUG_2, - "Got %d more bytes: %s", more, buffer); - + if(cmd->stdout > 0) { + memset(&buffer, 0, READ_MAX); + more = read(cmd->stdout, buffer, READ_MAX-1); + crm_trace("Got %d more bytes: %s", more, buffer); + } + if(more > 0) { crm_realloc(output, len + more + 1); sprintf(output+len, "%s", buffer); len += more; } } while (more == (READ_MAX-1) || (more < 0 && errno == EINTR)); if(cmd->stdout) { close(cmd->stdout); cmd->stdout = 0; } - while(rc != 0 && cmd->device_next) { - int exec_rc = 0; + if(rc != 0 && cmd->device_next) { stonith_device_t *dev = cmd->device_next->data; log_operation(cmd, rc, pid, dev->id, output); - cmd->device = dev->id; cmd->device_next = cmd->device_next->next; - - exec_rc = run_stonith_agent(dev->agent, cmd->action, cmd->victim, dev->params, dev->aliases, &rc, NULL, cmd); - if(exec_rc > 0) { - goto done; - } - pid = exec_rc; + schedule_stonith_command(cmd, dev); + goto done; } if(rc > 0) { rc = st_err_generic; } reply = stonith_construct_async_reply(cmd, output, data, rc); if(safe_str_eq(cmd->action, "metadata")) { /* Too verbose to log */ crm_free(output); output = NULL; } else if(crm_str_eq(cmd->action, "reboot", TRUE) || crm_str_eq(cmd->action, "poweroff", TRUE) || crm_str_eq(cmd->action, "poweron", TRUE) || crm_str_eq(cmd->action, "off", TRUE) || crm_str_eq(cmd->action, "on", TRUE)) { bcast = TRUE; } log_operation(cmd, rc, pid, NULL, output); crm_log_xml_debug_3(reply, "Reply"); if(bcast) { /* Send reply as T_STONITH_NOTIFY so everyone does notifications * Potentially limit to unsucessful operations to the originator? */ crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY); send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE); } else if(cmd->origin) { send_cluster_message(cmd->origin, crm_msg_stonith_ng, reply, FALSE); } else { do_local_reply(reply, cmd->client, cmd->options & st_opt_sync_call, FALSE); } free_async_command(cmd); done: reset_proctrack_data(proc); crm_free(output); free_xml(reply); free_xml(data); } static gint sort_device_priority(gconstpointer a, gconstpointer b) { const stonith_device_t *dev_a = a; const stonith_device_t *dev_b = a; if(dev_a->priority > dev_b->priority) { return -1; } else if(dev_a->priority < dev_b->priority) { return 1; } return 0; } static int stonith_fence(xmlNode *msg) { - int rc = 0; struct device_search_s search; stonith_device_t *device = NULL; async_command_t *cmd = create_async_command(msg, crm_element_value(msg, F_STONITH_ACTION)); xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_ERR); if(cmd == NULL) { return st_err_internal; } search.capable = NULL; search.host = crm_element_value(dev, F_STONITH_TARGET); crm_log_xml_info(msg, "Exec"); g_hash_table_foreach(device_list, search_devices, &search); crm_info("Found %d matching devices for '%s'", g_list_length(search.capable), search.host); if(g_list_length(search.capable) == 0) { free_async_command(cmd); return st_err_none_available; } /* Order based on priority */ search.capable = g_list_sort(search.capable, sort_device_priority); device = search.capable->data; cmd->device = device->id; if(g_list_length(search.capable) > 1) { cmd->device_list = search.capable; } - - return run_stonith_agent(device->agent, cmd->action, cmd->victim, device->params, device->aliases, &rc, NULL, cmd); + + schedule_stonith_command(cmd, device); + return stonith_pending; } xmlNode *stonith_construct_reply(xmlNode *request, char *output, xmlNode *data, int rc) { int lpc = 0; xmlNode *reply = NULL; const char *name = NULL; const char *value = NULL; const char *names[] = { F_STONITH_OPERATION, F_STONITH_CALLID, F_STONITH_CLIENTID, F_STONITH_REMOTE, F_STONITH_CALLOPTS }; crm_debug_4("Creating a basic reply"); reply = create_xml_node(NULL, T_STONITH_REPLY); crm_xml_add(reply, "st_origin", __FUNCTION__); crm_xml_add(reply, F_TYPE, T_STONITH_NG); crm_xml_add(reply, "st_output", output); crm_xml_add_int(reply, F_STONITH_RC, rc); CRM_CHECK(request != NULL, crm_warn("Can't create a sane reply"); return reply); for(lpc = 0; lpc < DIMOF(names); lpc++) { name = names[lpc]; value = crm_element_value(request, name); crm_xml_add(reply, name, value); } if(data != NULL) { crm_debug_4("Attaching reply output"); add_message_xml(reply, F_STONITH_CALLDATA, data); } return reply; } xmlNode *stonith_construct_async_reply(async_command_t *cmd, char *output, xmlNode *data, int rc) { xmlNode *reply = NULL; crm_debug_4("Creating a basic reply"); reply = create_xml_node(NULL, T_STONITH_REPLY); crm_xml_add(reply, "st_origin", __FUNCTION__); crm_xml_add(reply, F_TYPE, T_STONITH_NG); crm_xml_add(reply, F_STONITH_OPERATION, cmd->op); crm_xml_add(reply, F_STONITH_REMOTE, cmd->remote); crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client); crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id); crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options); crm_xml_add_int(reply, F_STONITH_RC, rc); crm_xml_add(reply, "st_output", output); if(data != NULL) { crm_info("Attaching reply output"); add_message_xml(reply, F_STONITH_CALLDATA, data); } return reply; } void stonith_command(stonith_client_t *client, xmlNode *request, const char *remote) { int call_options = 0; int rc = st_err_generic; gboolean is_reply = FALSE; gboolean always_reply = FALSE; xmlNode *reply = NULL; xmlNode *data = NULL; char *output = NULL; const char *op = crm_element_value(request, F_STONITH_OPERATION); const char *client_id = crm_element_value(request, F_STONITH_CLIENTID); crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options); if(get_xpath_object("//"T_STONITH_REPLY, request, LOG_DEBUG_3)) { is_reply = TRUE; } if(device_list == NULL) { device_list = g_hash_table_new_full( crm_str_hash, g_str_equal, NULL, free_device); } crm_debug("Processing %s%s from %s", op, is_reply?" reply":"", client?client->name:remote); if(crm_str_eq(op, CRM_OP_REGISTER, TRUE)) { return; } else if(crm_str_eq(op, STONITH_OP_DEVICE_ADD, TRUE)) { rc = stonith_device_register(request); do_stonith_notify(call_options, op, rc, request, NULL); } else if(crm_str_eq(op, STONITH_OP_DEVICE_DEL, TRUE)) { rc = stonith_device_remove(request); do_stonith_notify(call_options, op, rc, request, NULL); } else if(crm_str_eq(op, STONITH_OP_CONFIRM, TRUE)) { async_command_t *cmd = create_async_command(request, crm_element_value(request, F_STONITH_ACTION)); xmlNode *reply = stonith_construct_async_reply(cmd, NULL, NULL, 0); crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY); crm_notice("Broadcasting manual fencing confirmation for node %s", cmd->victim); send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE); free_async_command(cmd); free_xml(reply); } else if(crm_str_eq(op, STONITH_OP_EXEC, TRUE)) { rc = stonith_device_action(request, &output); } else if(is_reply && crm_str_eq(op, STONITH_OP_QUERY, TRUE)) { process_remote_stonith_query(request); return; } else if(crm_str_eq(op, STONITH_OP_QUERY, TRUE)) { create_remote_stonith_op(client_id, request, TRUE); /* Record it for the future notification */ rc = stonith_query(request, &data); always_reply = TRUE; } else if(is_reply && crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) { process_remote_stonith_exec(request); return; } else if(crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) { const char *flag_name = NULL; flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE); if(flag_name) { crm_debug("Setting %s callbacks for %s (%s): ON", flag_name, client->name, client->id); client->flags |= get_stonith_flag(flag_name); } flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE); if(flag_name) { crm_debug("Setting %s callbacks for %s (%s): off", flag_name, client->name, client->id); client->flags |= get_stonith_flag(flag_name); } return; /* } else if(is_reply && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) { */ /* process_remote_stonith_exec(request); */ /* return; */ } else if(is_reply == FALSE && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) { if(remote) { rc = stonith_fence(request); } else if(call_options & st_opt_local_first) { rc = stonith_fence(request); if(rc < 0) { initiate_remote_stonith_op(client, request); } } else { initiate_remote_stonith_op(client, request); } return; } else if (crm_str_eq(op, STONITH_OP_FENCE_HISTORY, TRUE)) { rc = stonith_fence_history(request, &data); always_reply = TRUE; } else { crm_err("Unknown %s%s from %s", op, is_reply?" reply":"", client?client->name:remote); crm_log_xml_warn(request, "UnknownOp"); } do_crm_log(rc>0?LOG_DEBUG:LOG_INFO,"Processed %s%s from %s: rc=%d", op, is_reply?" reply":"", client?client->name:remote, rc); - if(is_reply) { - /* Nothing */ + if(is_reply || rc == stonith_pending) { + /* Nothing (yet) */ } else if(remote) { reply = stonith_construct_reply(request, output, data, rc); send_cluster_message(remote, crm_msg_stonith_ng, reply, FALSE); free_xml(reply); } else if(rc <= 0 || always_reply) { reply = stonith_construct_reply(request, output, data, rc); do_local_reply(reply, client_id, call_options & st_opt_sync_call, remote!=NULL); free_xml(reply); } crm_free(output); free_xml(data); } diff --git a/fencing/internal.h b/fencing/internal.h index 94ac294119..16df0633fb 100644 --- a/fencing/internal.h +++ b/fencing/internal.h @@ -1,60 +1,63 @@ typedef struct stonith_device_s { char *id; char *agent; char *namespace; GListPtr targets; time_t targets_age; gboolean has_attr_map; guint priority; + guint active_pid; GHashTable *params; GHashTable *aliases; + GList *pending_ops; + crm_trigger_t *work; } stonith_device_t; typedef struct stonith_client_s { char *id; char *name; char *callback_id; const char *channel_name; IPC_Channel *channel; GCHSource *source; long long flags; } stonith_client_t; extern long long get_stonith_flag(const char *name); extern void stonith_command( stonith_client_t *client, xmlNode *op_request, const char *remote); extern void do_local_reply( xmlNode *notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer); extern xmlNode *stonith_construct_reply( xmlNode *request, char *output, xmlNode *data, int rc); extern xmlNode *stonith_construct_async_reply( async_command_t *cmd, char *output, xmlNode *data, int rc);; extern void do_stonith_notify( int options, const char *type, enum stonith_errors result, xmlNode *data, const char *remote); extern void initiate_remote_stonith_op(stonith_client_t *client, xmlNode *request); extern int process_remote_stonith_exec(xmlNode *msg); extern int process_remote_stonith_query(xmlNode *msg); extern void *create_remote_stonith_op(const char *client, xmlNode *request, gboolean peer); extern int stonith_fence_history(xmlNode *msg, xmlNode **output); extern char *stonith_our_uname; diff --git a/include/crm/stonith-ng.h b/include/crm/stonith-ng.h index 2f4f40c129..aa706f31cb 100644 --- a/include/crm/stonith-ng.h +++ b/include/crm/stonith-ng.h @@ -1,205 +1,209 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef STONITH_NG__H #define STONITH_NG__H #include #include #include enum stonith_state { stonith_connected_command, stonith_connected_query, stonith_disconnected, }; enum stonith_call_options { st_opt_none = 0x00000000, st_opt_verbose = 0x00000001, st_opt_allow_suicide = 0x00000002, st_opt_local_first = 0x00000004, st_opt_discard_reply = 0x00000010, st_opt_all_replies = 0x00000020, st_opt_scope_local = 0x00000100, st_opt_sync_call = 0x00001000, }; #define stonith_default_options = stonith_none enum stonith_errors { stonith_ok = 0, - st_err_generic = -1, - st_err_internal = -2, - st_err_not_supported = -3, - st_err_connection = -4, - st_err_missing = -5, - st_err_exists = -6, - st_err_timeout = -7, - st_err_ipc = -8, - st_err_peer = -9, - st_err_unknown_operation = -10, - st_err_unknown_device = -11, - st_err_unknown_port = -12, - st_err_none_available = -13, - st_err_authentication = -14, - st_err_signal = -15, + stonith_pending = -1, + st_err_generic = -2, + st_err_internal = -3, + st_err_not_supported = -4, + st_err_connection = -5, + st_err_missing = -6, + st_err_exists = -7, + st_err_timeout = -8, + st_err_ipc = -9, + st_err_peer = -10, + st_err_unknown_operation = -11, + st_err_unknown_device = -12, + st_err_unknown_port = -13, + st_err_none_available = -14, + st_err_authentication = -15, + st_err_signal = -16, + st_err_agent_fork = -17, + st_err_agent_args = -18, + st_err_agent = -19, }; #define F_STONITH_CLIENTID "st_clientid" #define F_STONITH_CALLOPTS "st_callopt" #define F_STONITH_CALLID "st_callid" #define F_STONITH_CALLDATA "st_calldata" #define F_STONITH_OPERATION "st_op" #define F_STONITH_TARGET "st_target" #define F_STONITH_REMOTE "st_remote_op" #define F_STONITH_RC "st_rc" #define F_STONITH_TIMEOUT "st_timeout" #define F_STONITH_CALLBACK_TOKEN "st_async_id" #define F_STONITH_CLIENTNAME "st_clientname" #define F_STONITH_NOTIFY_TYPE "st_notify_type" #define F_STONITH_NOTIFY_ACTIVATE "st_notify_activate" #define F_STONITH_NOTIFY_DEACTIVATE "st_notify_deactivate" #define F_STONITH_DELEGATE "st_delegate" #define F_STONITH_ORIGIN "st_origin" #define F_STONITH_HISTORY_LIST "st_history" #define F_STONITH_DATE "st_date" #define F_STONITH_STATE "st_state" #define T_STONITH_NG "stonith-ng" #define T_STONITH_REPLY "st-reply" #define F_STONITH_DEVICE "st_device_id" #define F_STONITH_ACTION "st_device_action" #define T_STONITH_NOTIFY "st_notify" #define T_STONITH_NOTIFY_DISCONNECT "st_notify_disconnect" #define STONITH_ATTR_ARGMAP "pcmk_arg_map" #define STONITH_ATTR_HOSTARG "pcmk_host_argument" #define STONITH_ATTR_HOSTMAP "pcmk_host_map" #define STONITH_ATTR_HOSTLIST "pcmk_host_list" #define STONITH_ATTR_HOSTCHECK "pcmk_host_check" #define STONITH_ATTR_ACTION_OP "option" /* To be replaced by 'action' at some point */ #define STONITH_OP_EXEC "st_execute" #define STONITH_OP_QUERY "st_query" #define STONITH_OP_FENCE "st_fence" #define STONITH_OP_CONFIRM "st_confirm" #define STONITH_OP_DEVICE_ADD "st_device_register" #define STONITH_OP_DEVICE_DEL "st_device_remove" #define STONITH_OP_DEVICE_METADATA "st_device_metadata" #define STONITH_OP_FENCE_HISTORY "st_fence_history" #define stonith_channel "st_command" #define stonith_channel_callback "st_callback" enum op_state { st_query, st_exec, st_done, st_failed, }; typedef struct stonith_key_value_s { char *key; char *value; struct stonith_key_value_s *next; } stonith_key_value_t; typedef struct stonith_history_s { char *target; char *action; char *origin; char *delegate; int completed; int state; struct stonith_history_s *next; } stonith_history_t; typedef struct stonith_s stonith_t; typedef struct stonith_api_operations_s { int (*free) (stonith_t *st); int (*connect) (stonith_t *st, const char *name, int *stonith_fd); int (*disconnect)(stonith_t *st); int (*remove_device)( stonith_t *st, int options, const char *name); int (*register_device)( stonith_t *st, int options, const char *id, const char *namespace, const char *agent, stonith_key_value_t *params); int (*metadata)(stonith_t *st, int options, const char *device, const char *namespace, char **output, int timeout); int (*list)(stonith_t *stonith, int call_options, const char *namespace, stonith_key_value_t **devices, int timeout); int (*call)(stonith_t *st, int options, const char *id, const char *action, const char *port, int timeout); int (*query)(stonith_t *st, int options, const char *node, stonith_key_value_t **devices, int timeout); int (*fence)(stonith_t *st, int options, const char *node, const char *action, int timeout); int (*confirm)(stonith_t *st, int options, const char *node); int (*history)(stonith_t *st, int options, const char *node, stonith_history_t **output, int timeout); int (*register_notification)( stonith_t *st, const char *event, void (*notify)(stonith_t *st, const char *event, xmlNode *msg)); int (*remove_notification)(stonith_t *st, const char *event); int (*register_callback)( stonith_t *st, int call_id, int timeout, bool only_success, void *userdata, const char *callback_name, void (*callback)(stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata)); int (*remove_callback)(stonith_t *st, int call_id, bool all_callbacks); } stonith_api_operations_t; struct stonith_s { enum stonith_state state; int call_id; int call_timeout; void *private; stonith_api_operations_t *cmds; }; /* Core functions */ extern stonith_t *stonith_api_new(void); extern void stonith_api_delete(stonith_t *st); extern const char *stonith_error2string(enum stonith_errors return_code); extern void stonith_dump_pending_callbacks(stonith_t *st); extern const char *get_stonith_provider(const char *agent, const char *provider); extern bool stonith_dispatch(stonith_t *st); extern stonith_key_value_t *stonith_key_value_add(stonith_key_value_t *kvp, const char *key, const char *value); extern void stonith_key_value_freeall(stonith_key_value_t *kvp, int keys, int values); #endif diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 21140e74b7..a2370b8d93 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1,1819 +1,1817 @@ /* * Copyright (c) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include /* Add it for compiling on OSX */ #include #include #include #include #include #include -#define FE_AGENT_FORK -2 -#define FE_AGENT_ERROR -3 - CRM_TRACE_INIT_DATA(stonith); typedef struct stonith_private_s { char *token; IPC_Channel *command_channel; IPC_Channel *callback_channel; GCHSource *callback_source; GHashTable *stonith_op_callback_table; GList *notify_list; void (*op_callback)( stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata); } stonith_private_t; typedef struct stonith_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*notify)(stonith_t *st, const char *event, xmlNode *msg); } stonith_notify_client_t; typedef struct stonith_callback_client_s { void (*callback)( stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata); const char *id; void *user_data; gboolean only_success; struct timer_rec_s *timer; } stonith_callback_client_t; struct notify_blob_s { stonith_t *stonith; xmlNode *xml; }; struct timer_rec_s { int call_id; int timeout; guint ref; stonith_t *stonith; }; typedef enum stonith_errors (*stonith_op_t)( const char *, int, const char *, xmlNode *, xmlNode*, xmlNode*, xmlNode**, xmlNode**); static const char META_TEMPLATE[] = "\n" "\n" "\n" " 1.0\n" " \n" "%s\n" " \n" " %s\n" "%s\n" " \n" " \n" " \n" " \n" " \n" " \n" " \n" " \n" " 2.0\n" " \n" "\n"; bool stonith_dispatch(stonith_t *st); gboolean stonith_dispatch_internal(IPC_Channel *channel, gpointer user_data); void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc); xmlNode *stonith_create_op( int call_id, const char *token, const char *op, xmlNode *data, int call_options); int stonith_send_command( stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options, int timeout); static void stonith_connection_destroy(gpointer user_data); static void stonith_send_notification(gpointer data, gpointer user_data); static void stonith_connection_destroy(gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *native = NULL; struct notify_blob_s blob; blob.stonith = stonith; blob.xml = create_xml_node(NULL, "notify"); native = stonith->private; native->callback_source = NULL; stonith->state = stonith_disconnected; crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); g_list_foreach(native->notify_list, stonith_send_notification, &blob); free_xml(blob.xml); } static int stonith_api_register_device( stonith_t *stonith, int call_options, const char *id, const char *namespace, const char *agent, stonith_key_value_t *params) { int rc = 0; xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE); xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); crm_xml_add(data, XML_ATTR_ID, id); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, "agent", agent); crm_xml_add(data, "namespace", namespace); for( ; params; params = params->next ) { hash2field( (gpointer)params->key, (gpointer)params->value, args ); } rc = stonith_send_command(stonith, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_remove_device( stonith_t *stonith, int call_options, const char *name) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, XML_ATTR_ID, name); rc = stonith_send_command(stonith, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0); free_xml(data); return rc; } static void append_arg( gpointer key, gpointer value, gpointer user_data) { int len = 3; /* =, \n, \0 */ int last = 0; char **args = user_data; CRM_CHECK(key != NULL, return); CRM_CHECK(value != NULL, return); if(strstr(key, "pcmk_")) { return; } else if(strstr(key, CRM_META)) { return; } else if(safe_str_eq(key, "crm_feature_set")) { return; } len += strlen(key); len += strlen(value); if(*args != NULL) { last = strlen(*args); } crm_realloc(*args, last+len); crm_debug_2("Appending: %s=%s", (char *)key, (char *)value); sprintf((*args)+last, "%s=%s\n", (char *)key, (char *)value); } static void append_const_arg(const char *key, const char *value, char **arg_list) { char *glib_sucks_key = crm_strdup(key); char *glib_sucks_value = crm_strdup(value); append_arg(glib_sucks_key, glib_sucks_value, arg_list); crm_free(glib_sucks_value); crm_free(glib_sucks_key); } static void append_host_specific_args(const char *victim, const char *map, GHashTable *params, char **arg_list) { char *name = NULL; int last = 0, lpc = 0, max = 0; if(map == NULL) { /* The best default there is for now... */ crm_debug("Using default arg map: port=uname"); append_const_arg("port", victim, arg_list); return; } max = strlen(map); crm_debug("Processing arg map: %s", map); for(; lpc < max + 1; lpc++) { if(isalpha(map[lpc])) { /* keep going */ } else if(map[lpc] == '=' || map[lpc] == ':') { crm_free(name); crm_malloc0(name, 1 + lpc - last); strncpy(name, map + last, lpc - last); crm_debug("Got name: %s", name); last = lpc + 1; } else if(map[lpc] == 0 || map[lpc] == ',' || isspace(map[lpc])) { char *param = NULL; const char *value = NULL; crm_malloc0(param, 1 + lpc - last); strncpy(param, map + last, lpc - last); last = lpc + 1; crm_debug("Got key: %s", param); if(name == NULL) { crm_err("Misparsed '%s', found '%s' without a name", map, param); crm_free(param); continue; } if(safe_str_eq(param, "uname")) { value = victim; } else { char *key = crm_meta_name(param); value = g_hash_table_lookup(params, key); crm_free(key); } if(value) { crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim); append_const_arg(name, value, arg_list); } else { crm_err("No node attribute '%s' for '%s'", name, victim); } crm_free(name); name=NULL; crm_free(param); if(map[lpc] == 0) { break; } } else if(isspace(map[lpc])) { last = lpc; } } crm_free(name); } static char *make_args(const char *action, const char *victim, GHashTable *device_args, GHashTable *port_map) { char buffer[512]; char *arg_list = NULL; const char *value = NULL; CRM_CHECK(action != NULL, return NULL); if(device_args) { g_hash_table_foreach(device_args, append_arg, &arg_list); } buffer[511] = 0; snprintf(buffer, 511, "pcmk_%s_action", action); if(device_args) { value = g_hash_table_lookup(device_args, buffer); } if(value == NULL && device_args) { /* Legacy support for early 1.1 releases - Remove for 1.2 */ snprintf(buffer, 511, "pcmk_%s_cmd", action); value = g_hash_table_lookup(device_args, buffer); } if(value) { crm_info("Substituting action '%s' for requested operation '%s'", value, action); action = value; } append_const_arg(STONITH_ATTR_ACTION_OP, action, &arg_list); if(victim && device_args) { const char *alias = victim; const char *param = g_hash_table_lookup(device_args, STONITH_ATTR_HOSTARG); if(port_map && g_hash_table_lookup(port_map, victim)) { alias = g_hash_table_lookup(port_map, victim); } /* Always supply the node's name too: * https://fedorahosted.org/cluster/wiki/FenceAgentAPI */ append_const_arg("nodename", victim, &arg_list); /* Check if we need to supply the victim in any other form */ if(param == NULL) { const char *map = g_hash_table_lookup(device_args, STONITH_ATTR_ARGMAP); if(map == NULL) { param = "port"; value = g_hash_table_lookup(device_args, param); } else { /* Legacy handling */ append_host_specific_args(alias, map, device_args, &arg_list); value = map; /* Nothing more to do */ } } else if(safe_str_eq(param, "none")) { value = param; /* Nothing more to do */ } else { value = g_hash_table_lookup(device_args, param); } /* Don't overwrite explictly set values for $param */ if(value == NULL || safe_str_eq(value, "dynamic")) { crm_info("%s-ing node '%s' as '%s=%s'", action, victim, param, alias); append_const_arg(param, alias, &arg_list); } } crm_debug_3("Calculated: %s", arg_list); return arg_list; } /* Borrowed from libfence and extended */ int run_stonith_agent( const char *agent, const char *action, const char *victim, GHashTable *device_args, GHashTable *port_map, int *agent_result, char **output, async_command_t *track) { char *args = make_args(action, victim, device_args, port_map); - int pid, status, len, rc = -1; + int pid, status, len, rc = st_err_internal; int p_read_fd, p_write_fd; /* parent read/write file descriptors */ int c_read_fd, c_write_fd; /* child read/write file descriptors */ int fd1[2]; int fd2[2]; c_read_fd = c_write_fd = p_read_fd = p_write_fd = -1; if (args == NULL || agent == NULL) goto fail; len = strlen(args); if (pipe(fd1)) goto fail; p_read_fd = fd1[0]; c_write_fd = fd1[1]; if (pipe(fd2)) goto fail; c_read_fd = fd2[0]; p_write_fd = fd2[1]; crm_debug("forking"); pid = fork(); if (pid < 0) { - *agent_result = FE_AGENT_FORK; + rc = st_err_agent_fork; goto fail; } if (pid) { /* parent */ int ret; int total = 0; fcntl(p_read_fd, F_SETFL, fcntl(p_read_fd, F_GETFL, 0) | O_NONBLOCK); do { crm_debug("sending args"); ret = write(p_write_fd, args+total, len-total); if(ret > 0) { total += ret; } } while (errno == EINTR && total < len); if (total != len) { crm_perror(LOG_ERR, "Sent %d not %d bytes", total, len); if(ret >= 0) { - rc = st_err_generic; + rc = st_err_agent_args; } goto fail; } close(p_write_fd); if(track) { track->stdout = p_read_fd; NewTrackedProc(pid, 0, PT_LOGNORMAL, track, track->pt_ops); crm_trace("Op: %s on %s, timeout: %d", action, agent, track->timeout); if(track->timeout) { track->killseq[0].mstimeout = track->timeout; /* after timeout send TERM */ track->killseq[0].signalno = SIGTERM; track->killseq[1].mstimeout = 5000; /* after another 5s remove it */ track->killseq[1].signalno = SIGKILL; track->killseq[2].mstimeout = 5000; /* if it's still there after another 5s, complain */ track->killseq[2].signalno = 0; SetTrackedProcTimeouts(pid, track->killseq); } else { crm_err("No timeout set for stonith operation %s with device %s", action, agent); } close(c_write_fd); close(c_read_fd); crm_free(args); return pid; } else { waitpid(pid, &status, 0); if(output != NULL) { len = 0; do { char buf[500]; ret = read(p_read_fd, buf, 500); if(ret > 0) { buf[ret] = 0; crm_realloc(*output, len + ret + 1); sprintf((*output)+len, "%s", buf); crm_debug("%d: %s", ret, (*output)+len); len += ret; } } while (ret == 500 || (ret < 0 && errno == EINTR)); } - - *agent_result = FE_AGENT_ERROR; + + rc = st_err_agent; + *agent_result = st_err_agent; if (WIFEXITED(status)) { crm_debug("result = %d", WEXITSTATUS(status)); *agent_result = -WEXITSTATUS(status); rc = 0; } } } else { /* child */ close(1); if (dup(c_write_fd) < 0) goto fail; close(2); if (dup(c_write_fd) < 0) goto fail; close(0); if (dup(c_read_fd) < 0) goto fail; /* keep c_write_fd open so parent can report all errors. */ close(c_read_fd); close(p_read_fd); close(p_write_fd); execlp(agent, agent, NULL); exit(EXIT_FAILURE); } fail: crm_free(args); if(p_read_fd >= 0) { close(p_read_fd); } if(p_write_fd >= 0) { close(p_write_fd); } if(c_read_fd >= 0) { close(c_read_fd); } if(c_write_fd >= 0) { close(c_write_fd); } - + return rc; } static int stonith_api_device_list( stonith_t *stonith, int call_options, const char *namespace, stonith_key_value_t **devices, int timeout) { int count = 0; if ( devices == NULL ) { crm_err("Parameter error: stonith_api_device_list"); return -2; } /* Include Heartbeat agents */ if(namespace == NULL || safe_str_eq("heartbeat", namespace)) { char **entry = NULL; char **type_list = stonith_types(); for(entry = type_list; entry != NULL && *entry; ++entry) { crm_trace("Added: %s", *entry); *devices = stonith_key_value_add(*devices, NULL, *entry); count++; } if(type_list) { stonith_free_hostlist(type_list); } } /* Include Red Hat agents, basically: ls -1 @sbin_dir@/fence_* */ if(namespace == NULL || safe_str_eq("redhat", namespace)) { struct dirent **namelist; int file_num = scandir(RH_STONITH_DIR, &namelist, 0, alphasort); if (file_num > 0) { struct stat prop; char buffer[FILENAME_MAX+1]; while (file_num--) { if ('.' == namelist[file_num]->d_name[0]) { free(namelist[file_num]); continue; } else if(0 != strncmp(RH_STONITH_PREFIX, namelist[file_num]->d_name, strlen(RH_STONITH_PREFIX))) { free(namelist[file_num]); continue; } snprintf(buffer,FILENAME_MAX,"%s/%s", RH_STONITH_DIR, namelist[file_num]->d_name); if(stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) { *devices = stonith_key_value_add(*devices, NULL, namelist[file_num]->d_name); count++; } free(namelist[file_num]); } free(namelist); } } return count; } static int stonith_api_device_metadata( stonith_t *stonith, int call_options, const char *agent, const char *namespace, char **output, int timeout) { int rc = 0; int bufferlen = 0; char *buffer = NULL; char *xml_meta_longdesc = NULL; char *xml_meta_shortdesc = NULL; char *meta_param = NULL; char *meta_longdesc = NULL; char *meta_shortdesc = NULL; const char *provider = get_stonith_provider(agent, namespace); Stonith *stonith_obj = NULL; static const char *no_parameter_info = ""; crm_info("looking up %s/%s metadata", agent, provider); /* By having this in a library, we can access it from stonith_admin * when neither lrmd or stonith-ng are running * Important for the crm shell's validations... */ if(safe_str_eq(provider, "redhat")) { int exec_rc = run_stonith_agent( agent, "metadata", NULL, NULL, NULL, &rc, &buffer, NULL); if(exec_rc < 0 || rc != 0 || buffer == NULL) { /* failed */ crm_debug("Query failed: %d %d: %s", exec_rc, rc, crm_str(buffer)); /* provide a fake metadata entry */ meta_longdesc = crm_strdup(no_parameter_info); meta_shortdesc = crm_strdup(no_parameter_info); meta_param = crm_strdup( " \n" " \n" " \n" " \n" " Fencing action (null, off, on, [reboot], status, hostlist, devstatus)\n" " \n" " "); goto build; } } else { stonith_obj = stonith_new(agent); meta_longdesc = crm_strdup(stonith_get_info(stonith_obj, ST_DEVICEDESCR)); if (meta_longdesc == NULL) { crm_warn("no long description in %s's metadata.", agent); meta_longdesc = crm_strdup(no_parameter_info); } meta_shortdesc = crm_strdup(stonith_get_info(stonith_obj, ST_DEVICEID)); if (meta_shortdesc == NULL) { crm_warn("no short description in %s's metadata.", agent); meta_shortdesc = crm_strdup(no_parameter_info); } meta_param = crm_strdup(stonith_get_info(stonith_obj, ST_CONF_XML)); if (meta_param == NULL) { crm_warn("no list of parameters in %s's metadata.", agent); meta_param = crm_strdup(no_parameter_info); } build: xml_meta_longdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_longdesc); xml_meta_shortdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_shortdesc); bufferlen = strlen(META_TEMPLATE) + strlen(agent) + strlen(xml_meta_longdesc) + strlen(xml_meta_shortdesc) + strlen(meta_param) + 1; crm_malloc0(buffer, bufferlen); snprintf(buffer, bufferlen-1, META_TEMPLATE, agent, xml_meta_longdesc, xml_meta_shortdesc, meta_param); xmlFree(xml_meta_longdesc); xmlFree(xml_meta_shortdesc); if(stonith_obj) { stonith_delete(stonith_obj); } crm_free(meta_shortdesc); crm_free(meta_longdesc); crm_free(meta_param); } if(output) { *output = buffer; } else { crm_free(buffer); } return rc; } static int stonith_api_confirm( stonith_t *stonith, int call_options, const char *target) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, target); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_query( stonith_t *stonith, int call_options, const char *target, stonith_key_value_t **devices, int timeout) { int rc = 0, lpc = 0, max = 0; xmlNode *data = NULL; xmlNode *output = NULL; xmlXPathObjectPtr xpathObj = NULL; CRM_CHECK(devices != NULL, return st_err_missing); data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, target); rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout); if(rc < 0) { return rc; } xpathObj = xpath_search(output, "//@agent"); if(xpathObj) { max = xpathObj->nodesetval->nodeNr; for(lpc = 0; lpc < max; lpc++) { xmlNode *match = getXpathResult(xpathObj, lpc); CRM_CHECK(match != NULL, continue); crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match)); *devices = stonith_key_value_add(*devices, NULL, crm_element_value(match, XML_ATTR_ID)); } } free_xml(output); free_xml(data); return max; } static int stonith_api_call( stonith_t *stonith, int call_options, const char *id, const char *action, const char *victim, int timeout) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, F_STONITH_DEVICE, id); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add(data, F_STONITH_TARGET, victim); rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, NULL, call_options, timeout); free_xml(data); return rc; } static int stonith_api_fence( stonith_t *stonith, int call_options, const char *node, const char *action, int timeout) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); return rc; } static int stonith_api_history( stonith_t *stonith, int call_options, const char *node, stonith_history_t **history, int timeout) { int rc = 0; xmlNode *data = NULL; xmlNode *output = NULL; stonith_history_t *last = NULL; *history = NULL; if(node) { data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); } rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output, call_options | st_opt_sync_call, timeout); free_xml(data); if (rc == 0) { xmlNode *op = NULL; xmlNode *reply = get_xpath_object("//"F_STONITH_HISTORY_LIST, output, LOG_ERR); for(op = __xml_first_child(reply); op != NULL; op = __xml_next(op)) { stonith_history_t *kvp; crm_malloc0(kvp, sizeof(stonith_history_t)); kvp->target = crm_element_value_copy(op, F_STONITH_TARGET); kvp->action = crm_element_value_copy(op, F_STONITH_ACTION); kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN); kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE); crm_element_value_int(op, F_STONITH_DATE, &kvp->completed); crm_element_value_int(op, F_STONITH_STATE, &kvp->state); if (last) { last->next = kvp; } else { *history = kvp; } last = kvp; } } return rc; } const char * stonith_error2string(enum stonith_errors return_code) { const char *error_msg = NULL; switch(return_code) { case stonith_ok: error_msg = "OK"; break; case st_err_not_supported: error_msg = "Not supported"; break; case st_err_authentication: error_msg = "Not authenticated"; break; case st_err_generic: error_msg = "Generic error"; break; case st_err_internal: error_msg = "Internal error"; break; case st_err_unknown_device: error_msg = "Unknown device"; break; case st_err_unknown_operation: error_msg = "Unknown operation"; break; case st_err_unknown_port: error_msg = "Unknown victim"; break; case st_err_none_available: error_msg = "No available fencing devices"; break; case st_err_connection: error_msg = "Not connected"; break; case st_err_missing: error_msg = "Missing input"; break; case st_err_exists: error_msg = "Device exists"; break; case st_err_timeout: error_msg = "Operation timed out"; break; case st_err_signal: error_msg = "Killed by signal"; break; case st_err_ipc: error_msg = "IPC connection failed"; break; case st_err_peer: error_msg = "Error from peer"; break; } if(error_msg == NULL) { crm_err("Unknown Stonith error code: %d", return_code); error_msg = ""; } return error_msg; } gboolean is_redhat_agent(const char *agent) { int rc = 0; struct stat prop; char buffer[FILENAME_MAX+1]; snprintf(buffer,FILENAME_MAX,"%s/%s", RH_STONITH_DIR, agent); rc = stat(buffer, &prop); if (rc >= 0 && S_ISREG(prop.st_mode)) { return TRUE; } return FALSE; } const char *get_stonith_provider(const char *agent, const char *provider) { /* This function sucks */ if(is_redhat_agent(agent)) { return "redhat"; } else { Stonith *stonith_obj = stonith_new(agent); if(stonith_obj) { stonith_delete(stonith_obj); return "heartbeat"; } } crm_err("No such device: %s", agent); return NULL; } static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const stonith_notify_client_t *a_client = a; const stonith_notify_client_t *b_client = b; CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if(rc == 0) { if(a_client->notify == NULL || b_client->notify == NULL) { return 0; } else if(a_client->notify == b_client->notify) { return 0; } else if(((long)a_client->notify) < ((long)b_client->notify)) { crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return -1; } crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return 1; } return rc; } static int get_stonith_token(IPC_Channel *ch, char **token) { int rc = stonith_ok; xmlNode *reg_msg = NULL; const char *msg_type = NULL; const char *tmp_ticket = NULL; CRM_CHECK(ch != NULL, return st_err_missing); CRM_CHECK(token != NULL, return st_err_missing); crm_debug_4("Waiting for msg on command channel"); reg_msg = xmlfromIPC(ch, MAX_IPC_DELAY); if(ch->ops->get_chan_status(ch) != IPC_CONNECT) { crm_err("No reply message - disconnected"); free_xml(reg_msg); return st_err_connection; } else if(reg_msg == NULL) { crm_err("No reply message - empty"); return st_err_ipc; } msg_type = crm_element_value(reg_msg, F_STONITH_OPERATION); tmp_ticket = crm_element_value(reg_msg, F_STONITH_CLIENTID); if(safe_str_neq(msg_type, CRM_OP_REGISTER) ) { crm_err("Invalid registration message: %s", msg_type); rc = st_err_internal; } else if(tmp_ticket == NULL) { crm_err("No registration token provided"); crm_log_xml_warn(reg_msg, "Bad reply"); rc = st_err_internal; } else { crm_debug("Obtained registration token: %s", tmp_ticket); *token = crm_strdup(tmp_ticket); } free_xml(reg_msg); return rc; } xmlNode *stonith_create_op( int call_id, const char *token, const char *op, xmlNode *data, int call_options) { int rc = HA_OK; xmlNode *op_msg = create_xml_node(NULL, "stonith_command"); CRM_CHECK(op_msg != NULL, return NULL); CRM_CHECK(token != NULL, return NULL); crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command"); crm_xml_add(op_msg, F_TYPE, T_STONITH_NG); crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token); crm_xml_add(op_msg, F_STONITH_OPERATION, op); crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id); crm_debug_4("Sending call options: %.8lx, %d", (long)call_options, call_options); crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options); if(data != NULL) { add_message_xml(op_msg, F_STONITH_CALLDATA, data); } if (rc != HA_OK) { crm_err("Failed to create STONITH operation message"); crm_log_xml(LOG_ERR, "BadOp", op_msg); free_xml(op_msg); return NULL; } return op_msg; } static void stonith_destroy_op_callback(gpointer data) { stonith_callback_client_t *blob = data; if(blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } crm_free(blob->timer); crm_free(blob); } static int stonith_api_signoff(stonith_t* stonith) { stonith_private_t *native = stonith->private; crm_debug("Signing out of the STONITH Service"); /* close channels */ if (native->command_channel != NULL) { native->command_channel->ops->destroy( native->command_channel); native->command_channel = NULL; } if (native->callback_source != NULL) { G_main_del_IPC_Channel(native->callback_source); native->callback_source = NULL; } if (native->callback_channel != NULL) { #ifdef BUG native->callback_channel->ops->destroy( native->callback_channel); #endif native->callback_channel = NULL; } stonith->state = stonith_disconnected; return stonith_ok; } static int stonith_api_signon( stonith_t* stonith, const char *name, int *stonith_fd) { int rc = stonith_ok; xmlNode *hello = NULL; char *uuid_ticket = NULL; stonith_private_t *native = stonith->private; crm_debug_4("Connecting command channel"); stonith->state = stonith_connected_command; native->command_channel = init_client_ipc_comms_nodispatch(stonith_channel); if(native->command_channel == NULL) { crm_debug("Connection to command channel failed"); rc = st_err_connection; } else if(native->command_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = st_err_authentication; } if(rc == stonith_ok) { rc = get_stonith_token(native->command_channel, &uuid_ticket); if(rc == stonith_ok) { native->token = uuid_ticket; uuid_ticket = NULL; } else { stonith->state = stonith_disconnected; native->command_channel->ops->disconnect(native->command_channel); return rc; } } native->callback_channel = init_client_ipc_comms_nodispatch( stonith_channel_callback); if(native->callback_channel == NULL) { crm_debug("Connection to callback channel failed"); rc = st_err_connection; } else if(native->callback_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = st_err_authentication; } if(rc == stonith_ok) { native->callback_channel->send_queue->max_qlen = 500; rc = get_stonith_token(native->callback_channel, &uuid_ticket); if(rc == stonith_ok) { crm_free(native->token); native->token = uuid_ticket; } } if(rc == stonith_ok) { CRM_CHECK(native->token != NULL, ;); hello = stonith_create_op(0, native->token, CRM_OP_REGISTER, NULL, 0); crm_xml_add(hello, F_STONITH_CLIENTNAME, name); if(send_ipc_message(native->command_channel, hello) == FALSE) { rc = st_err_internal; } free_xml(hello); } if(rc == stonith_ok) { if(stonith_fd != NULL) { *stonith_fd = native->callback_channel->ops->get_recv_select_fd( native->callback_channel); } else { /* do mainloop */ crm_debug_4("Connecting callback channel"); native->callback_source = G_main_add_IPC_Channel( G_PRIORITY_HIGH, native->callback_channel, FALSE, stonith_dispatch_internal, stonith, default_ipc_connection_destroy); if(native->callback_source == NULL) { crm_err("Callback source not recorded"); rc = st_err_connection; } else { set_IPC_Channel_dnotify( native->callback_source, stonith_connection_destroy); } } } if(rc == stonith_ok) { #if HAVE_MSGFROMIPC_TIMEOUT stonith->call_timeout = MAX_IPC_DELAY; #endif crm_debug("Connection to STONITH successful"); return stonith_ok; } crm_debug("Connection to STONITH failed: %s", stonith_error2string(rc)); stonith->cmds->disconnect(stonith); return rc; } static int stonith_set_notification(stonith_t* stonith, const char *callback, int enabled) { xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__); stonith_private_t *native = stonith->private; if(stonith->state != stonith_disconnected) { crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY); if(enabled) { crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback); } else { crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback); } send_ipc_message(native->callback_channel, notify_msg); } free_xml(notify_msg); return stonith_ok; } static int stonith_api_add_notification( stonith_t *stonith, const char *event, void (*callback)(stonith_t *stonith, const char *event, xmlNode *msg)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; private = stonith->private; crm_debug_2("Adding callback for %s events (%d)", event, g_list_length(private->notify_list)); crm_malloc0(new_client, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = callback; list_item = g_list_find_custom( private->notify_list, new_client, stonithlib_GCompareFunc); if(list_item != NULL) { crm_warn("Callback already present"); crm_free(new_client); return st_err_exists; } else { private->notify_list = g_list_append(private->notify_list, new_client); stonith_set_notification(stonith, event, 1); crm_debug_3("Callback added (%d)", g_list_length(private->notify_list)); } return stonith_ok; } static int stonith_api_del_notification(stonith_t *stonith, const char *event) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; crm_debug("Removing callback for %s events", event); private = stonith->private; crm_malloc0(new_client, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = NULL; list_item = g_list_find_custom( private->notify_list, new_client, stonithlib_GCompareFunc); stonith_set_notification(stonith, event, 0); if(list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; private->notify_list = g_list_remove(private->notify_list, list_client); crm_free(list_client); crm_debug_3("Removed callback"); } else { crm_debug_3("Callback not present"); } crm_free(new_client); return stonith_ok; } static gboolean stonith_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_debug("Async call %d timed out after %dms", timer->call_id, timer->timeout); stonith_perform_callback(timer->stonith, NULL, timer->call_id, st_err_timeout); /* Always return TRUE, never remove the handler * We do that in stonith_del_callback() */ return TRUE; } static int stonith_api_add_callback( stonith_t *stonith, int call_id, int timeout, bool only_success, void *user_data, const char *callback_name, void (*callback)( stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata)) { stonith_callback_client_t *blob = NULL; stonith_private_t *private = NULL; CRM_CHECK(stonith != NULL, return st_err_missing); CRM_CHECK(stonith->private != NULL, return st_err_missing); private = stonith->private; if(call_id == 0) { private->op_callback = callback; } else if(call_id < 0) { if(only_success == FALSE) { callback(stonith, NULL, call_id, call_id, NULL, user_data); } else { crm_warn("STONITH call failed: %s", stonith_error2string(call_id)); } return FALSE; } crm_malloc0(blob, sizeof(stonith_callback_client_t)); blob->id = callback_name; blob->only_success = only_success; blob->user_data = user_data; blob->callback = callback; if(timeout > 0) { struct timer_rec_s *async_timer = NULL; crm_malloc0(async_timer, sizeof(struct timer_rec_s)); blob->timer = async_timer; async_timer->stonith = stonith; async_timer->call_id = call_id; async_timer->timeout = timeout*1100; async_timer->ref = g_timeout_add( async_timer->timeout, stonith_async_timeout_handler, async_timer); } g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob); return TRUE; } static int stonith_api_del_callback(stonith_t *stonith, int call_id, bool all_callbacks) { stonith_private_t *private = stonith->private; if(all_callbacks) { private->op_callback = NULL; g_hash_table_destroy(private->stonith_op_callback_table); private->stonith_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); } else if(call_id == 0) { private->op_callback = NULL; } else { g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); } return stonith_ok; } static void stonith_dump_pending_op( gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); stonith_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, crm_str(blob->id)); } void stonith_dump_pending_callbacks(stonith_t *stonith) { stonith_private_t *private = stonith->private; if(private->stonith_op_callback_table == NULL) { return; } return g_hash_table_foreach( private->stonith_op_callback_table, stonith_dump_pending_op, NULL); } void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc) { xmlNode *output = NULL; stonith_private_t *private = NULL; stonith_callback_client_t *blob = NULL; stonith_callback_client_t local_blob; CRM_CHECK(stonith != NULL, return); CRM_CHECK(stonith->private != NULL, return); private = stonith->private; local_blob.id = NULL; local_blob.callback = NULL; local_blob.user_data = NULL; local_blob.only_success = FALSE; if(msg != NULL) { crm_element_value_int(msg, F_STONITH_RC, &rc); crm_element_value_int(msg, F_STONITH_CALLID, &call_id); output = get_message_xml(msg, F_STONITH_CALLDATA); } CRM_CHECK(call_id > 0, crm_warn("Strange or missing call-id")); blob = g_hash_table_lookup( private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); if(blob != NULL) { local_blob = *blob; blob = NULL; stonith_api_del_callback(stonith, call_id, FALSE); } else { crm_debug_2("No callback found for call %d", call_id); local_blob.callback = NULL; } if(stonith == NULL) { crm_debug("No stonith object supplied"); } if(local_blob.callback != NULL && (rc == stonith_ok || local_blob.only_success == FALSE)) { crm_debug_2("Invoking callback %s for call %d", crm_str(local_blob.id), call_id); local_blob.callback(stonith, msg, call_id, rc, output, local_blob.user_data); } else if(private->op_callback == NULL && rc != stonith_ok) { crm_warn("STONITH command failed: %s", stonith_error2string(rc)); crm_log_xml(LOG_DEBUG, "Failed STONITH Update", msg); } if(private->op_callback != NULL) { crm_debug_2("Invoking global callback for call %d", call_id); private->op_callback(stonith, msg, call_id, rc, output, NULL); } crm_debug_4("OP callback activated."); } static void stonith_send_notification(gpointer data, gpointer user_data) { struct notify_blob_s *blob = user_data; stonith_notify_client_t *entry = data; const char *event = NULL; if(blob->xml == NULL) { crm_warn("Skipping callback - NULL message"); return; } event = crm_element_value(blob->xml, F_SUBTYPE); if(entry == NULL) { crm_warn("Skipping callback - NULL callback client"); return; } else if(entry->notify == NULL) { crm_warn("Skipping callback - NULL callback"); return; } else if(safe_str_neq(entry->event, event)) { crm_debug_4("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } crm_debug_4("Invoking callback for %p/%s event...", entry, event); entry->notify(blob->stonith, event, blob->xml); crm_debug_4("Callback invoked..."); } int stonith_send_command( stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options, int timeout) { int rc = HA_OK; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; stonith_private_t *native = stonith->private; if(stonith->state == stonith_disconnected) { return st_err_connection; } if(output_data != NULL) { *output_data = NULL; } if(op == NULL) { crm_err("No operation specified"); return st_err_missing; } stonith->call_id++; /* prevent call_id from being negative (or zero) and conflicting * with the stonith_errors enum * use 2 because we use it as (stonith->call_id - 1) below */ if(stonith->call_id < 1) { stonith->call_id = 1; } CRM_CHECK(native->token != NULL, ;); op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options); if(op_msg == NULL) { return st_err_missing; } crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout); crm_debug_3("Sending %s message to STONITH service, Timeout: %d", op, timeout); if(send_ipc_message(native->command_channel, op_msg) == FALSE) { crm_err("Sending message to STONITH service FAILED"); free_xml(op_msg); return st_err_ipc; } else { crm_debug_3("Message sent"); } free_xml(op_msg); if((call_options & st_opt_discard_reply)) { crm_debug_3("Discarding reply"); return stonith_ok; } else if(!(call_options & st_opt_sync_call)) { crm_debug_3("Async call, returning"); CRM_CHECK(stonith->call_id != 0, return st_err_ipc); return stonith->call_id; } rc = IPC_OK; crm_debug_3("Waiting for a syncronous reply"); rc = stonith_ok; while(IPC_ISRCONN(native->command_channel)) { int reply_id = -1; int msg_id = stonith->call_id; op_reply = xmlfromIPC(native->command_channel, timeout); if(op_reply == NULL) { rc = st_err_peer; break; } crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id); if(reply_id <= 0) { rc = st_err_peer; break; } else if(reply_id == msg_id) { crm_debug_3("Syncronous reply received"); crm_log_xml(LOG_MSG, "Reply", op_reply); if(crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) { rc = st_err_peer; } if(output_data != NULL && is_not_set(call_options, st_opt_discard_reply)) { *output_data = op_reply; op_reply = NULL; } goto done; } else if(reply_id < msg_id) { crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else if((reply_id - 10000) > msg_id) { /* wrap-around case */ crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else { crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id); } free_xml(op_reply); op_reply = NULL; } if(op_reply == NULL && stonith->state == stonith_disconnected) { rc = st_err_connection; } else if(rc == stonith_ok && op_reply == NULL) { rc = st_err_peer; } done: if(IPC_ISRCONN(native->command_channel) == FALSE) { crm_err("STONITH disconnected: %d", native->command_channel->ch_status); stonith->state = stonith_disconnected; } free_xml(op_reply); return rc; } static gboolean stonith_msgready(stonith_t* stonith) { stonith_private_t *private = NULL; if (stonith == NULL) { crm_err("No STONITH!"); return FALSE; } private = stonith->private; if(private->command_channel != NULL) { /* drain the channel */ IPC_Channel *cmd_ch = private->command_channel; xmlNode *cmd_msg = NULL; while(cmd_ch->ch_status != IPC_DISCONNECT && cmd_ch->ops->is_message_pending(cmd_ch)) { /* this will happen when the STONITH exited from beneath us */ cmd_msg = xmlfromIPC(cmd_ch, MAX_IPC_DELAY); free_xml(cmd_msg); } } else { crm_err("No command channel"); } if(private->callback_channel == NULL) { crm_err("No callback channel"); return FALSE; } else if(private->callback_channel->ch_status == IPC_DISCONNECT) { crm_info("Lost connection to the STONITH service [%d].", private->callback_channel->farside_pid); return FALSE; } else if(private->callback_channel->ops->is_message_pending( private->callback_channel)) { crm_debug_4("Message pending on command channel [%d]", private->callback_channel->farside_pid); return TRUE; } crm_debug_3("No message pending"); return FALSE; } static int stonith_rcvmsg(stonith_t* stonith) { const char *type = NULL; stonith_private_t *private = NULL; struct notify_blob_s blob; if (stonith == NULL) { crm_err("No STONITH!"); return FALSE; } blob.stonith = stonith; private = stonith->private; /* if it is not blocking mode and no message in the channel, return */ if (stonith_msgready(stonith) == FALSE) { crm_debug_3("No message ready and non-blocking..."); return 0; } /* IPC_INTR is not a factor here */ blob.xml = xmlfromIPC(private->callback_channel, MAX_IPC_DELAY); if (blob.xml == NULL) { crm_warn("Received a NULL msg from STONITH service."); return 0; } /* do callbacks */ type = crm_element_value(blob.xml, F_TYPE); crm_debug_4("Activating %s callbacks...", type); if(safe_str_eq(type, T_STONITH_NG)) { stonith_perform_callback(stonith, blob.xml, 0, 0); } else if(safe_str_eq(type, T_STONITH_NOTIFY)) { g_list_foreach(private->notify_list, stonith_send_notification, &blob); } else { crm_err("Unknown message type: %s", type); crm_log_xml_warn(blob.xml, "BadReply"); } free_xml(blob.xml); return 1; } bool stonith_dispatch(stonith_t *st) { stonith_private_t *private = NULL; bool stay_connected = TRUE; CRM_CHECK(st != NULL, return FALSE); private = st->private; stay_connected = stonith_dispatch_internal(private->callback_channel, (gpointer *)st); return stay_connected; } gboolean stonith_dispatch_internal(IPC_Channel *channel, gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *private = NULL; gboolean stay_connected = TRUE; CRM_CHECK(stonith != NULL, return FALSE); private = stonith->private; CRM_CHECK(private->callback_channel == channel, return FALSE); while(stonith_msgready(stonith)) { /* invoke the callbacks but dont block */ int rc = stonith_rcvmsg(stonith); if( rc < 0) { crm_err("Message acquisition failed: %d", rc); break; } else if(rc == 0) { break; } } if(private->callback_channel && private->callback_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the STONITH service [%d/callback].", channel->farside_pid); private->callback_source = NULL; stay_connected = FALSE; } if(private->command_channel && private->command_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the STONITH service [%d/command].", channel->farside_pid); private->callback_source = NULL; stay_connected = FALSE; } return stay_connected; } static int stonith_api_free (stonith_t* stonith) { int rc = stonith_ok; if(stonith->state != stonith_disconnected) { rc = stonith->cmds->disconnect(stonith); } if(stonith->state == stonith_disconnected) { stonith_private_t *private = stonith->private; g_hash_table_destroy(private->stonith_op_callback_table); crm_free(private->token); crm_free(stonith->private); crm_free(stonith->cmds); crm_free(stonith); } return rc; } void stonith_api_delete(stonith_t *stonith) { stonith_private_t* private = stonith->private; GList *list = private->notify_list; while(list != NULL) { stonith_notify_client_t *client = g_list_nth_data(list, 0); list = g_list_remove(list, client); crm_free(client); } stonith->cmds->free(stonith); stonith = NULL; } stonith_t *stonith_api_new(void) { stonith_t* new_stonith = NULL; stonith_private_t* private = NULL; crm_malloc0(new_stonith, sizeof(stonith_t)); crm_malloc0(private, sizeof(stonith_private_t)); new_stonith->private = private; private->stonith_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); private->notify_list = NULL; new_stonith->call_id = 1; new_stonith->state = stonith_disconnected; crm_malloc0(new_stonith->cmds, sizeof(stonith_api_operations_t)); new_stonith->cmds->free = stonith_api_free; new_stonith->cmds->connect = stonith_api_signon; new_stonith->cmds->disconnect = stonith_api_signoff; new_stonith->cmds->call = stonith_api_call; new_stonith->cmds->fence = stonith_api_fence; new_stonith->cmds->confirm = stonith_api_confirm; new_stonith->cmds->history = stonith_api_history; new_stonith->cmds->list = stonith_api_device_list; new_stonith->cmds->metadata = stonith_api_device_metadata; new_stonith->cmds->query = stonith_api_query; new_stonith->cmds->remove_device = stonith_api_remove_device; new_stonith->cmds->register_device = stonith_api_register_device; new_stonith->cmds->remove_callback = stonith_api_del_callback; new_stonith->cmds->register_callback = stonith_api_add_callback; new_stonith->cmds->remove_notification = stonith_api_del_notification; new_stonith->cmds->register_notification = stonith_api_add_notification; return new_stonith; } stonith_key_value_t *stonith_key_value_add(stonith_key_value_t *kvp, const char *key, const char *value) { stonith_key_value_t *p; crm_malloc(p, sizeof(stonith_key_value_t)); p->next = kvp; p->key = crm_strdup(key); p->value = crm_strdup(value); return( p ); } void stonith_key_value_freeall(stonith_key_value_t *kvp, int keys, int values) { stonith_key_value_t *p; while(kvp) { p = kvp->next; if(keys) { free(kvp->key); } if(values) { free(kvp->value); } free(kvp); kvp = p; } }