diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
index b12b20dc10..c03fa0bd4c 100644
--- a/crmd/lrm_state.c
+++ b/crmd/lrm_state.c
@@ -1,749 +1,749 @@
/*
* Copyright (C) 2012 David Vossel <dvossel@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crmd.h>
#include <crmd_fsa.h>
#include <crmd_messages.h>
#include <crmd_callbacks.h>
#include <crmd_lrm.h>
GHashTable *lrm_state_table = NULL;
extern GHashTable *proxy_table;
int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
static void
free_rsc_info(gpointer value)
{
lrmd_rsc_info_t *rsc_info = value;
lrmd_free_rsc_info(rsc_info);
}
static void
free_deletion_op(gpointer value)
{
struct pending_deletion_op_s *op = value;
free(op->rsc);
delete_ha_msg_input(op->input);
free(op);
}
static void
free_recurring_op(gpointer value)
{
struct recurring_op_s *op = (struct recurring_op_s *)value;
free(op->user_data);
free(op->rsc_id);
free(op->op_type);
free(op->op_key);
if (op->params) {
g_hash_table_destroy(op->params);
}
free(op);
}
static gboolean
fail_pending_op(gpointer key, gpointer value, gpointer user_data)
{
lrmd_event_data_t event = { 0, };
lrm_state_t *lrm_state = user_data;
struct recurring_op_s *op = (struct recurring_op_s *)value;
crm_trace("Pre-emptively failing %s_%s_%d on %s (call=%s, %s)",
op->rsc_id, op->op_type, op->interval,
lrm_state->node_name, key, op->user_data);
event.type = lrmd_event_exec_complete;
event.rsc_id = op->rsc_id;
event.op_type = op->op_type;
event.user_data = op->user_data;
event.timeout = 0;
event.interval = op->interval;
event.rc = PCMK_OCF_CONNECTION_DIED;
event.op_status = PCMK_LRM_OP_ERROR;
event.t_run = op->start_time;
event.t_rcchange = op->start_time;
event.t_rcchange = op->start_time;
event.call_id = op->call_id;
event.remote_nodename = lrm_state->node_name;
event.params = op->params;
process_lrm_event(lrm_state, &event, op);
return TRUE;
}
gboolean
lrm_state_is_local(lrm_state_t *lrm_state)
{
if (lrm_state == NULL || fsa_our_uname == NULL) {
return FALSE;
}
if (strcmp(lrm_state->node_name, fsa_our_uname) != 0) {
return FALSE;
}
return TRUE;
}
lrm_state_t *
lrm_state_create(const char *node_name)
{
lrm_state_t *state = NULL;
if (!node_name) {
crm_err("No node name given for lrm state object");
return NULL;
}
state = calloc(1, sizeof(lrm_state_t));
if (!state) {
return NULL;
}
state->node_name = strdup(node_name);
state->rsc_info_cache = g_hash_table_new_full(crm_str_hash,
g_str_equal, NULL, free_rsc_info);
state->deletion_ops = g_hash_table_new_full(crm_str_hash,
g_str_equal, g_hash_destroy_str, free_deletion_op);
state->pending_ops = g_hash_table_new_full(crm_str_hash,
g_str_equal, g_hash_destroy_str, free_recurring_op);
state->resource_history = g_hash_table_new_full(crm_str_hash,
g_str_equal, NULL, history_free);
g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
return state;
}
void
lrm_state_destroy(const char *node_name)
{
g_hash_table_remove(lrm_state_table, node_name);
}
static gboolean
remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
{
remote_proxy_t *proxy = value;
const char *node_name = user_data;
if (safe_str_eq(node_name, proxy->node_name)) {
return TRUE;
}
return FALSE;
}
static void
internal_lrm_state_destroy(gpointer data)
{
lrm_state_t *lrm_state = data;
if (!lrm_state) {
return;
}
crm_trace("Destroying proxy table %s with %d members", lrm_state->node_name, g_hash_table_size(proxy_table));
g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
remote_ra_cleanup(lrm_state);
lrmd_api_delete(lrm_state->conn);
if (lrm_state->rsc_info_cache) {
crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache));
g_hash_table_destroy(lrm_state->rsc_info_cache);
}
if (lrm_state->resource_history) {
crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history));
g_hash_table_destroy(lrm_state->resource_history);
}
if (lrm_state->deletion_ops) {
crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops));
g_hash_table_destroy(lrm_state->deletion_ops);
}
if (lrm_state->pending_ops) {
crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops));
g_hash_table_destroy(lrm_state->pending_ops);
}
free((char *)lrm_state->node_name);
free(lrm_state);
}
void
lrm_state_reset_tables(lrm_state_t * lrm_state)
{
if (lrm_state->resource_history) {
crm_trace("Re-setting history op cache with %d members",
g_hash_table_size(lrm_state->resource_history));
g_hash_table_remove_all(lrm_state->resource_history);
}
if (lrm_state->deletion_ops) {
crm_trace("Re-setting deletion op cache with %d members",
g_hash_table_size(lrm_state->deletion_ops));
g_hash_table_remove_all(lrm_state->deletion_ops);
}
if (lrm_state->pending_ops) {
crm_trace("Re-setting pending op cache with %d members",
g_hash_table_size(lrm_state->pending_ops));
g_hash_table_remove_all(lrm_state->pending_ops);
}
if (lrm_state->rsc_info_cache) {
crm_trace("Re-setting rsc info cache with %d members",
g_hash_table_size(lrm_state->rsc_info_cache));
g_hash_table_remove_all(lrm_state->rsc_info_cache);
}
}
gboolean
lrm_state_init_local(void)
{
if (lrm_state_table) {
return TRUE;
}
lrm_state_table =
g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, internal_lrm_state_destroy);
if (!lrm_state_table) {
return FALSE;
}
proxy_table =
g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free);
if (!proxy_table) {
g_hash_table_destroy(lrm_state_table);
return FALSE;
}
return TRUE;
}
void
lrm_state_destroy_all(void)
{
if (lrm_state_table) {
crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table));
g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
}
if(proxy_table) {
crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
g_hash_table_destroy(proxy_table); proxy_table = NULL;
}
}
lrm_state_t *
lrm_state_find(const char *node_name)
{
if (!node_name) {
return NULL;
}
return g_hash_table_lookup(lrm_state_table, node_name);
}
lrm_state_t *
lrm_state_find_or_create(const char *node_name)
{
lrm_state_t *lrm_state;
lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
if (!lrm_state) {
lrm_state = lrm_state_create(node_name);
}
return lrm_state;
}
GList *
lrm_state_get_list(void)
{
return g_hash_table_get_values(lrm_state_table);
}
void
lrm_state_disconnect(lrm_state_t * lrm_state)
{
int removed = 0;
if (!lrm_state->conn) {
return;
}
crm_trace("Disconnecting %s", lrm_state->node_name);
((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
if (is_not_set(fsa_input_register, R_SHUTDOWN)) {
removed = g_hash_table_foreach_remove(lrm_state->pending_ops, fail_pending_op, lrm_state);
crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name);
}
lrmd_api_delete(lrm_state->conn);
lrm_state->conn = NULL;
}
int
lrm_state_is_connected(lrm_state_t * lrm_state)
{
if (!lrm_state->conn) {
return FALSE;
}
return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
}
int
lrm_state_poke_connection(lrm_state_t * lrm_state)
{
if (!lrm_state->conn) {
return -1;
}
return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
}
int
lrm_state_ipc_connect(lrm_state_t * lrm_state)
{
int ret;
if (!lrm_state->conn) {
lrm_state->conn = lrmd_api_new();
((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback);
}
ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);
if (ret != pcmk_ok) {
lrm_state->num_lrm_register_fails++;
} else {
lrm_state->num_lrm_register_fails = 0;
}
return ret;
}
static int
remote_proxy_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
{
/* Async responses from cib and friends back to clients via pacemaker_remoted */
xmlNode *xml = NULL;
remote_proxy_t *proxy = userdata;
lrm_state_t *lrm_state = lrm_state_find(proxy->node_name);
uint32_t flags;
if (lrm_state == NULL) {
return 0;
}
xml = string2xml(buffer);
if (xml == NULL) {
crm_warn("Received a NULL msg from IPC service.");
return 1;
}
flags = crm_ipc_buffer_flags(proxy->ipc);
if (flags & crm_ipc_proxied_relay_response) {
crm_trace("Passing response back to %.8s on %s: %.200s - request id: %d", proxy->session_id, proxy->node_name, buffer, proxy->last_request_id);
remote_proxy_relay_response(lrm_state->conn, proxy->session_id, xml, proxy->last_request_id);
proxy->last_request_id = 0;
} else {
crm_trace("Passing event back to %.8s on %s: %.200s", proxy->session_id, proxy->node_name, buffer);
remote_proxy_relay_event(lrm_state->conn, proxy->session_id, xml);
}
free_xml(xml);
return 1;
}
static void
remote_proxy_disconnected(void *userdata)
{
remote_proxy_t *proxy = userdata;
lrm_state_t *lrm_state = lrm_state_find(proxy->node_name);
crm_trace("Destroying %s (%p)", lrm_state->node_name, userdata);
proxy->source = NULL;
proxy->ipc = NULL;
if (lrm_state && lrm_state->conn) {
remote_proxy_notify_destroy(lrm_state->conn, proxy->session_id);
}
g_hash_table_remove(proxy_table, proxy->session_id);
}
static remote_proxy_t *
remote_proxy_new(const char *node_name, const char *session_id, const char *channel)
{
static struct ipc_client_callbacks proxy_callbacks = {
.dispatch = remote_proxy_dispatch_internal,
.destroy = remote_proxy_disconnected
};
remote_proxy_t *proxy = calloc(1, sizeof(remote_proxy_t));
proxy->node_name = strdup(node_name);
proxy->session_id = strdup(session_id);
if (safe_str_eq(channel, CRM_SYSTEM_CRMD)) {
proxy->is_local = TRUE;
} else {
proxy->source = mainloop_add_ipc_client(channel, G_PRIORITY_LOW, 0, proxy, &proxy_callbacks);
proxy->ipc = mainloop_get_ipc_client(proxy->source);
if (proxy->source == NULL) {
remote_proxy_free(proxy);
return NULL;
}
}
crm_trace("created proxy session ID %s", proxy->session_id);
g_hash_table_insert(proxy_table, proxy->session_id, proxy);
return proxy;
}
gboolean
crmd_is_proxy_session(const char *session)
{
return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
}
void
crmd_proxy_send(const char *session, xmlNode *msg)
{
remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
lrm_state_t *lrm_state = NULL;
if (!proxy) {
return;
}
crm_log_xml_trace(msg, "to-proxy");
lrm_state = lrm_state_find(proxy->node_name);
if (lrm_state) {
crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
remote_proxy_relay_event(lrm_state->conn, session, msg);
}
}
static void
crmd_proxy_dispatch(const char *session, xmlNode *msg)
{
crm_log_xml_trace(msg, "CRMd-PROXY[inbound]");
crm_xml_add(msg, F_CRM_SYS_FROM, session);
if (crmd_authorize_message(msg, NULL, session)) {
route_message(C_IPC_MESSAGE, msg);
}
trigger_fsa(fsa_source);
}
static void
remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg)
{
lrm_state_t *lrm_state = userdata;
const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
int msg_id = 0;
/* sessions are raw ipc connections to IPC,
* all we do is proxy requests/responses exactly
* like they are given to us at the ipc level. */
CRM_CHECK(op != NULL, return);
CRM_CHECK(session != NULL, return);
crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);
/* This is msg from remote ipc client going to real ipc server */
if (safe_str_eq(op, "new")) {
const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);
CRM_CHECK(channel != NULL, return);
if (remote_proxy_new(lrm_state->node_name, session, channel) == NULL) {
remote_proxy_notify_destroy(lrmd, session);
}
- crm_info("new remote proxy client established to %s, session id %s", channel, session);
+ crm_trace("new remote proxy client established to %s, session id %s", channel, session);
} else if (safe_str_eq(op, "destroy")) {
remote_proxy_end_session(session);
} else if (safe_str_eq(op, "request")) {
int flags = 0;
xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
const char *name = crm_element_value(msg, F_LRMD_IPC_CLIENT);
remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
CRM_CHECK(request != NULL, return);
if (proxy == NULL) {
/* proxy connection no longer exists */
remote_proxy_notify_destroy(lrmd, session);
return;
} else if ((proxy->is_local == FALSE) && (crm_ipc_connected(proxy->ipc) == FALSE)) {
remote_proxy_end_session(session);
return;
}
proxy->last_request_id = 0;
crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");
#if ENABLE_ACL
CRM_ASSERT(lrm_state->node_name);
crm_acl_get_set_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
#endif
if (proxy->is_local) {
/* this is for the crmd, which we are, so don't try
* and connect/send to ourselves over ipc. instead
* do it directly. */
crmd_proxy_dispatch(session, request);
if (flags & crm_ipc_client_response) {
xmlNode *op_reply = create_xml_node(NULL, "ack");
crm_xml_add(op_reply, "function", __FUNCTION__);
crm_xml_add_int(op_reply, "line", __LINE__);
remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
free_xml(op_reply);
}
} else if(is_set(flags, crm_ipc_proxied)) {
const char *type = crm_element_value(request, F_TYPE);
int rc = 0;
if (safe_str_eq(type, T_ATTRD)
&& crm_element_value(request, F_ATTRD_HOST) == NULL) {
crm_xml_add(request, F_ATTRD_HOST, proxy->node_name);
crm_xml_add_int(request, F_ATTRD_HOST_ID, get_local_nodeid(0));
}
rc = crm_ipc_send(proxy->ipc, request, flags, 5000, NULL);
if(rc < 0) {
xmlNode *op_reply = create_xml_node(NULL, "nack");
crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);
/* Send a n'ack so the caller doesn't block */
crm_xml_add(op_reply, "function", __FUNCTION__);
crm_xml_add_int(op_reply, "line", __LINE__);
crm_xml_add_int(op_reply, "rc", rc);
remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
free_xml(op_reply);
} else {
crm_trace("Relayed %s request %d from %s to %s for %s",
op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
proxy->last_request_id = msg_id;
}
} else {
int rc = pcmk_ok;
xmlNode *op_reply = NULL;
/* For backwards compatibility with pacemaker_remoted <= 1.1.10 */
crm_trace("Relaying %s request %d from %s to %s for %s",
op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
rc = crm_ipc_send(proxy->ipc, request, flags, 10000, &op_reply);
if(rc < 0) {
crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);
} else {
crm_trace("Relayed %s request %d from %s to %s for %s",
op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
}
if(op_reply) {
remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
free_xml(op_reply);
}
}
} else {
crm_err("Unknown proxy operation: %s", op);
}
}
int
lrm_state_remote_connect_async(lrm_state_t * lrm_state, const char *server, int port,
int timeout_ms)
{
int ret;
if (!lrm_state->conn) {
lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port);
if (!lrm_state->conn) {
return -1;
}
((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback);
lrmd_internal_set_proxy_callback(lrm_state->conn, lrm_state, remote_proxy_cb);
}
crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms);
ret =
((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name,
timeout_ms);
if (ret != pcmk_ok) {
lrm_state->num_lrm_register_fails++;
} else {
lrm_state->num_lrm_register_fails = 0;
}
return ret;
}
int
lrm_state_get_metadata(lrm_state_t * lrm_state,
const char *class,
const char *provider,
const char *agent, char **output, enum lrmd_call_options options)
{
if (!lrm_state->conn) {
return -ENOTCONN;
}
/* Optimize this... only retrieve metadata from local lrmd connection. Perhaps consider
* caching result. */
return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata(lrm_state->conn, class, provider, agent,
output, options);
}
int
lrm_state_cancel(lrm_state_t * lrm_state, const char *rsc_id, const char *action, int interval)
{
if (!lrm_state->conn) {
return -ENOTCONN;
}
/* Optimize this, cancel requires a synced request/response to the server.
* Figure out a way to make this async. */
if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
return remote_ra_cancel(lrm_state, rsc_id, action, interval);
}
return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id, action, interval);
}
lrmd_rsc_info_t *
lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
{
lrmd_rsc_info_t *rsc = NULL;
if (!lrm_state->conn) {
return NULL;
}
if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
return remote_ra_get_rsc_info(lrm_state, rsc_id);
}
rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
if (rsc == NULL) {
/* only contact the lrmd if we don't already have a cached rsc info */
rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
if (rsc == NULL) {
return NULL;
}
/* cache the result */
g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
}
return lrmd_copy_rsc_info(rsc);
}
int
lrm_state_exec(lrm_state_t * lrm_state, const char *rsc_id, const char *action, const char *userdata, int interval, /* ms */
int timeout, /* ms */
int start_delay, /* ms */
lrmd_key_value_t * params)
{
if (!lrm_state->conn) {
lrmd_key_value_freeall(params);
return -ENOTCONN;
}
if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
return remote_ra_exec(lrm_state,
rsc_id, action, userdata, interval, timeout, start_delay, params);
}
return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn,
rsc_id,
action,
userdata,
interval,
timeout,
start_delay,
lrmd_opt_notify_changes_only, params);
}
int
lrm_state_register_rsc(lrm_state_t * lrm_state,
const char *rsc_id,
const char *class,
const char *provider, const char *agent, enum lrmd_call_options options)
{
if (!lrm_state->conn) {
return -ENOTCONN;
}
/* optimize this... this function is a synced round trip from client to daemon.
* The crmd/lrm.c code path should be re-factored to allow the register of resources
* to be performed async. The lrmd client api needs to make an async version
* of register available. */
if (is_remote_lrmd_ra(agent, provider, NULL)) {
return lrm_state_find_or_create(rsc_id) ? pcmk_ok : -1;
}
return ((lrmd_t *) lrm_state->conn)->cmds->register_rsc(lrm_state->conn, rsc_id, class,
provider, agent, options);
}
int
lrm_state_unregister_rsc(lrm_state_t * lrm_state,
const char *rsc_id, enum lrmd_call_options options)
{
if (!lrm_state->conn) {
return -ENOTCONN;
}
/* optimize this... this function is a synced round trip from client to daemon.
* The crmd/lrm.c code path that uses this function should always treat it as an
* async operation. The lrmd client api needs to make an async version unreg available. */
if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
lrm_state_destroy(rsc_id);
return pcmk_ok;
}
g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);
return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
}
diff --git a/lrmd/ipc_proxy.c b/lrmd/ipc_proxy.c
index 72d83c4c75..9427393c8f 100644
--- a/lrmd/ipc_proxy.c
+++ b/lrmd/ipc_proxy.c
@@ -1,411 +1,411 @@
/*
* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#include <crm_internal.h>
#include <lrmd_private.h>
#include <glib.h>
#include <unistd.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/services.h>
#include <crm/common/mainloop.h>
#include <crm/common/ipc.h>
#include <crm/common/ipcs.h>
#include <crm/cib/internal.h>
#include <crm/fencing/internal.h>
static qb_ipcs_service_t *cib_ro = NULL;
static qb_ipcs_service_t *cib_rw = NULL;
static qb_ipcs_service_t *cib_shm = NULL;
static qb_ipcs_service_t *attrd_ipcs = NULL;
static qb_ipcs_service_t *crmd_ipcs = NULL;
static qb_ipcs_service_t *stonith_ipcs = NULL;
/* ipc providers == crmd clients connecting from cluster nodes */
GHashTable *ipc_providers;
/* ipc clients == things like cibadmin, crm_resource, connecting locally */
GHashTable *ipc_clients;
static int32_t
ipc_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid, const char *ipc_channel)
{
void *key = NULL;
void *value = NULL;
crm_client_t *client;
crm_client_t *ipc_proxy = NULL;
GHashTableIter iter;
xmlNode *msg;
crm_trace("Connection %p on channel %s", c, ipc_channel);
if (g_hash_table_size(ipc_providers) == 0) {
crm_err("No ipc providers available for uid %d gid %d", uid, gid);
return -EREMOTEIO;
}
g_hash_table_iter_init(&iter, ipc_providers);
if (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
/* grab the first provider available, any provider in this
* table will work. Usually there will only be one. These are
* lrmd client connections originating for a cluster node's crmd. */
ipc_proxy = value;
} else {
crm_err("No ipc providers available for uid %d gid %d", uid, gid);
return -EREMOTEIO;
}
/* this new client is a local ipc client on a remote
* guest wanting to access the ipc on any available cluster nodes */
client = crm_client_new(c, uid, gid);
if (client == NULL) {
return -EREMOTEIO;
}
/* This ipc client is bound to a single ipc provider. If the
* provider goes away, this client is disconnected */
client->userdata = strdup(ipc_proxy->id);
client->name = crm_strdup_printf("proxy-%s-%d-%.8s", ipc_channel, client->pid, client->id);
g_hash_table_insert(ipc_clients, client->id, client);
msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
crm_xml_add(msg, F_LRMD_IPC_OP, "new");
crm_xml_add(msg, F_LRMD_IPC_IPC_SERVER, ipc_channel);
crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
lrmd_server_send_notify(ipc_proxy, msg);
free_xml(msg);
crm_debug("created new ipc proxy with session id %s", client->id);
return 0;
}
static int32_t
crmd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
return ipc_proxy_accept(c, uid, gid, CRM_SYSTEM_CRMD);
}
static int32_t
attrd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
return ipc_proxy_accept(c, uid, gid, T_ATTRD);
}
static int32_t
stonith_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
return ipc_proxy_accept(c, uid, gid, "stonith-ng");
}
static int32_t
cib_proxy_accept_rw(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
return ipc_proxy_accept(c, uid, gid, cib_channel_rw);
}
static int32_t
cib_proxy_accept_ro(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
{
return ipc_proxy_accept(c, uid, gid, cib_channel_ro);
}
static void
ipc_proxy_created(qb_ipcs_connection_t * c)
{
crm_trace("Connection %p", c);
}
void
ipc_proxy_forward_client(crm_client_t *ipc_proxy, xmlNode *xml)
{
const char *session = crm_element_value(xml, F_LRMD_IPC_SESSION);
const char *msg_type = crm_element_value(xml, F_LRMD_IPC_OP);
xmlNode *msg = get_message_xml(xml, F_LRMD_IPC_MSG);
crm_client_t *ipc_client = crm_client_get_by_id(session);
int rc = 0;
if (ipc_client == NULL) {
xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
crm_xml_add(msg, F_LRMD_IPC_OP, "destroy");
crm_xml_add(msg, F_LRMD_IPC_SESSION, session);
lrmd_server_send_notify(ipc_proxy, msg);
free_xml(msg);
return;
}
/* This is an event or response from the ipc provider
* going to the local ipc client.
*
* Looking at the chain of events.
*
* -----remote node----------------|---- cluster node ------
* ipc_client <--1--> this code <--2--> crmd:remote_proxy_cb/remote_proxy_relay_event() <----3----> ipc server
*
* This function is receiving a msg from connection 2
* and forwarding it to connection 1.
*/
if (safe_str_eq(msg_type, "event")) {
- crm_info("Sending event to %s", ipc_client->id);
+ crm_trace("Sending event to %s", ipc_client->id);
rc = crm_ipcs_send(ipc_client, 0, msg, crm_ipc_server_event);
} else if (safe_str_eq(msg_type, "response")) {
int msg_id = 0;
crm_element_value_int(xml, F_LRMD_IPC_MSG_ID, &msg_id);
- crm_info("Sending response to %d - %s", ipc_client->request_id, ipc_client->id);
+ crm_trace("Sending response to %d - %s", ipc_client->request_id, ipc_client->id);
rc = crm_ipcs_send(ipc_client, msg_id, msg, FALSE);
CRM_LOG_ASSERT(msg_id == ipc_client->request_id);
ipc_client->request_id = 0;
} else if (safe_str_eq(msg_type, "destroy")) {
qb_ipcs_disconnect(ipc_client->ipcs);
} else {
crm_err("Unknown ipc proxy msg type %s" , msg_type);
}
if (rc < 0) {
crm_warn("IPC Proxy send to ipc client %s failed, rc = %d", ipc_client->id, rc);
}
}
static int32_t
ipc_proxy_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
{
uint32_t id = 0;
uint32_t flags = 0;
crm_client_t *client = crm_client_get(c);
crm_client_t *ipc_proxy = crm_client_get_by_id(client->userdata);
xmlNode *request = NULL;
xmlNode *msg = NULL;
if (!ipc_proxy) {
qb_ipcs_disconnect(client->ipcs);
return 0;
}
/* This is a request from the local ipc client going
* to the ipc provider.
*
* Looking at the chain of events.
*
* -----remote node----------------|---- cluster node ------
* ipc_client <--1--> this code <--2--> crmd:remote_proxy_dispatch_internal() <----3----> ipc server
*
* This function is receiving a request from connection
* 1 and forwarding it to connection 2.
*/
request = crm_ipcs_recv(client, data, size, &id, &flags);
if (!request) {
return 0;
}
CRM_CHECK(client != NULL, crm_err("Invalid client");
return FALSE);
CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
return FALSE);
/* this ensures that synced request/responses happen over the event channel
* in the crmd, allowing the crmd to process the messages async */
set_bit(flags, crm_ipc_proxied);
client->request_id = id;
msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
crm_xml_add(msg, F_LRMD_IPC_OP, "request");
crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
crm_xml_add(msg, F_LRMD_IPC_CLIENT, crm_client_name(client));
crm_xml_add(msg, F_LRMD_IPC_USER, client->user);
crm_xml_add_int(msg, F_LRMD_IPC_MSG_ID, id);
crm_xml_add_int(msg, F_LRMD_IPC_MSG_FLAGS, flags);
add_message_xml(msg, F_LRMD_IPC_MSG, request);
lrmd_server_send_notify(ipc_proxy, msg);
free_xml(msg);
return 0;
}
static int32_t
ipc_proxy_closed(qb_ipcs_connection_t * c)
{
crm_client_t *client = crm_client_get(c);
crm_client_t *ipc_proxy;
if (client == NULL) {
return 0;
}
ipc_proxy = crm_client_get_by_id(client->userdata);
crm_trace("Connection %p", c);
if (ipc_proxy) {
xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
crm_xml_add(msg, F_LRMD_IPC_OP, "destroy");
crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
lrmd_server_send_notify(ipc_proxy, msg);
free_xml(msg);
}
g_hash_table_remove(ipc_clients, client->id);
free(client->userdata);
client->userdata = NULL;
crm_client_destroy(client);
return 0;
}
static void
ipc_proxy_destroy(qb_ipcs_connection_t * c)
{
crm_trace("Connection %p", c);
ipc_proxy_closed(c);
}
static struct qb_ipcs_service_handlers crmd_proxy_callbacks = {
.connection_accept = crmd_proxy_accept,
.connection_created = ipc_proxy_created,
.msg_process = ipc_proxy_dispatch,
.connection_closed = ipc_proxy_closed,
.connection_destroyed = ipc_proxy_destroy
};
static struct qb_ipcs_service_handlers attrd_proxy_callbacks = {
.connection_accept = attrd_proxy_accept,
.connection_created = ipc_proxy_created,
.msg_process = ipc_proxy_dispatch,
.connection_closed = ipc_proxy_closed,
.connection_destroyed = ipc_proxy_destroy
};
static struct qb_ipcs_service_handlers stonith_proxy_callbacks = {
.connection_accept = stonith_proxy_accept,
.connection_created = ipc_proxy_created,
.msg_process = ipc_proxy_dispatch,
.connection_closed = ipc_proxy_closed,
.connection_destroyed = ipc_proxy_destroy
};
static struct qb_ipcs_service_handlers cib_proxy_callbacks_ro = {
.connection_accept = cib_proxy_accept_ro,
.connection_created = ipc_proxy_created,
.msg_process = ipc_proxy_dispatch,
.connection_closed = ipc_proxy_closed,
.connection_destroyed = ipc_proxy_destroy
};
static struct qb_ipcs_service_handlers cib_proxy_callbacks_rw = {
.connection_accept = cib_proxy_accept_rw,
.connection_created = ipc_proxy_created,
.msg_process = ipc_proxy_dispatch,
.connection_closed = ipc_proxy_closed,
.connection_destroyed = ipc_proxy_destroy
};
void
ipc_proxy_add_provider(crm_client_t *ipc_proxy)
{
if (ipc_providers == NULL) {
return;
}
g_hash_table_insert(ipc_providers, ipc_proxy->id, ipc_proxy);
}
void
ipc_proxy_remove_provider(crm_client_t *ipc_proxy)
{
GHashTableIter iter;
crm_client_t *ipc_client = NULL;
char *key = NULL;
GList *remove_these = NULL;
GListPtr gIter = NULL;
if (ipc_providers == NULL) {
return;
}
g_hash_table_remove(ipc_providers, ipc_proxy->id);
g_hash_table_iter_init(&iter, ipc_clients);
while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & ipc_client)) {
const char *proxy_id = ipc_client->userdata;
if (safe_str_eq(proxy_id, ipc_proxy->id)) {
crm_info("ipc proxy connection for client %s pid %d destroyed because cluster node disconnected.",
ipc_client->id, ipc_client->pid);
/* we can't remove during the iteration, so copy items
* to a list we can destroy later */
remove_these = g_list_append(remove_these, ipc_client);
}
}
for (gIter = remove_these; gIter != NULL; gIter = gIter->next) {
ipc_client = gIter->data;
qb_ipcs_disconnect(ipc_client->ipcs);
}
/* just frees the list, not the elements in the list */
g_list_free(remove_these);
}
void
ipc_proxy_init(void)
{
ipc_clients = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
ipc_providers = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
cib_ipc_servers_init(&cib_ro,
&cib_rw,
&cib_shm,
&cib_proxy_callbacks_ro,
&cib_proxy_callbacks_rw);
attrd_ipc_server_init(&attrd_ipcs, &attrd_proxy_callbacks);
stonith_ipc_server_init(&stonith_ipcs, &stonith_proxy_callbacks);
crmd_ipcs = crmd_ipc_server_init(&crmd_proxy_callbacks);
if (crmd_ipcs == NULL) {
crm_err("Failed to create crmd server: exiting and inhibiting respawn.");
crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
crm_exit(DAEMON_RESPAWN_STOP);
}
}
void
ipc_proxy_cleanup(void)
{
if (ipc_providers) {
g_hash_table_destroy(ipc_providers);
}
if (ipc_clients) {
g_hash_table_destroy(ipc_clients);
}
cib_ipc_servers_destroy(cib_ro, cib_rw, cib_shm);
qb_ipcs_destroy(attrd_ipcs);
qb_ipcs_destroy(stonith_ipcs);
qb_ipcs_destroy(crmd_ipcs);
cib_ro = NULL;
cib_rw = NULL;
cib_shm = NULL;
ipc_providers = NULL;
ipc_clients = NULL;
}