diff --git a/attrd/commands.c b/attrd/commands.c
index 108db51dc7..985f90c8d3 100644
--- a/attrd/commands.c
+++ b/attrd/commands.c
@@ -1,775 +1,770 @@
/*
* Copyright (C) 2013 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <glib.h>
#include <crm/msg_xml.h>
#include <crm/cluster.h>
#include <crm/cib.h>
#include <crm/cluster/internal.h>
#include <crm/cluster/election.h>
#include <crm/cib/internal.h>
#include <internal.h>
#define ATTRD_PROTOCOL_VERSION "1"
int last_cib_op_done = 0;
char *peer_writer = NULL;
GHashTable *attributes = NULL;
typedef struct attribute_s {
char *uuid; /* TODO: Remove if at all possible */
char *id;
char *set;
GHashTable *values;
int update;
int timeout_ms;
bool changed;
bool unknown_peer_uuids;
mainloop_timer_t *timer;
char *user;
} attribute_t;
typedef struct attribute_value_s {
uint32_t nodeid;
gboolean is_remote;
char *nodename;
char *current;
char *requested;
char *stored;
} attribute_value_t;
void write_attribute(attribute_t *a);
void write_or_elect_attribute(attribute_t *a);
void attrd_peer_update(crm_node_t *peer, xmlNode *xml, bool filter);
void attrd_peer_sync(crm_node_t *peer, xmlNode *xml);
void attrd_peer_remove(const char *host, const char *source);
static gboolean
send_attrd_message(crm_node_t * node, xmlNode * data)
{
crm_xml_add(data, F_TYPE, T_ATTRD);
crm_xml_add(data, F_ATTRD_IGNORE_LOCALLY, "atomic-version"); /* Tell older versions to ignore our messages */
crm_xml_add(data, F_ATTRD_VERSION, ATTRD_PROTOCOL_VERSION);
crm_xml_add_int(data, F_ATTRD_WRITER, election_state(writer));
return send_cluster_message(node, crm_msg_attrd, data, TRUE);
}
static gboolean
attribute_timer_cb(gpointer data)
{
attribute_t *a = data;
crm_trace("Dampen interval expired for %s in state %d", a->id, election_state(writer));
write_or_elect_attribute(a);
return FALSE;
}
static void
free_attribute_value(gpointer data)
{
attribute_value_t *v = data;
free(v->nodename);
free(v->current);
free(v->requested);
free(v->stored);
free(v);
}
void
free_attribute(gpointer data)
{
attribute_t *a = data;
if(a) {
free(a->id);
free(a->set);
free(a->uuid);
free(a->user);
mainloop_timer_del(a->timer);
g_hash_table_destroy(a->values);
free(a);
}
}
xmlNode *
build_attribute_xml(
xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int timeout_ms, const char *user,
const char *peer, uint32_t peerid, const char *value)
{
xmlNode *xml = create_xml_node(parent, __FUNCTION__);
crm_xml_add(xml, F_ATTRD_ATTRIBUTE, name);
crm_xml_add(xml, F_ATTRD_SET, set);
crm_xml_add(xml, F_ATTRD_KEY, uuid);
crm_xml_add(xml, F_ATTRD_USER, user);
crm_xml_add(xml, F_ATTRD_HOST, peer);
crm_xml_add_int(xml, F_ATTRD_HOST_ID, peerid);
crm_xml_add(xml, F_ATTRD_VALUE, value);
crm_xml_add_int(xml, F_ATTRD_DAMPEN, timeout_ms/1000);
return xml;
}
static attribute_t *
create_attribute(xmlNode *xml)
{
int dampen = 0;
const char *value = crm_element_value(xml, F_ATTRD_DAMPEN);
attribute_t *a = calloc(1, sizeof(attribute_t));
a->id = crm_element_value_copy(xml, F_ATTRD_ATTRIBUTE);
a->set = crm_element_value_copy(xml, F_ATTRD_SET);
a->uuid = crm_element_value_copy(xml, F_ATTRD_KEY);
a->values = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute_value);
#if ENABLE_ACL
crm_trace("Performing all %s operations as user '%s'", a->id, a->user);
a->user = crm_element_value_copy(xml, F_ATTRD_USER);
#endif
if(value) {
dampen = crm_get_msec(value);
crm_trace("Created attribute %s with delay %dms (%s)", a->id, dampen, value);
} else {
crm_trace("Created attribute %s with no delay", a->id);
}
if(dampen > 0) {
a->timeout_ms = dampen;
a->timer = mainloop_timer_add(strdup(a->id), a->timeout_ms, FALSE, attribute_timer_cb, a);
}
g_hash_table_replace(attributes, a->id, a);
return a;
}
void
attrd_client_message(crm_client_t *client, xmlNode *xml)
{
bool broadcast = FALSE;
static int plus_plus_len = 5;
const char *op = crm_element_value(xml, F_ATTRD_TASK);
if(safe_str_eq(op, "peer-remove")) {
const char *host = crm_element_value(xml, F_ATTRD_HOST);
crm_info("Client %s is requesting all values for %s be removed", client->name, host);
if(host) {
broadcast = TRUE;
}
} else if(safe_str_eq(op, "update")) {
attribute_t *a = NULL;
attribute_value_t *v = NULL;
char *key = crm_element_value_copy(xml, F_ATTRD_KEY);
char *set = crm_element_value_copy(xml, F_ATTRD_SET);
char *host = crm_element_value_copy(xml, F_ATTRD_HOST);
const char *attr = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
const char *value = crm_element_value(xml, F_ATTRD_VALUE);
a = g_hash_table_lookup(attributes, attr);
if(host == NULL) {
crm_trace("Inferring host");
host = strdup(attrd_cluster->uname);
crm_xml_add(xml, F_ATTRD_HOST, host);
crm_xml_add_int(xml, F_ATTRD_HOST_ID, attrd_cluster->nodeid);
}
if (value) {
int offset = 1;
int int_value = 0;
int value_len = strlen(value);
if (value_len < (plus_plus_len + 2)
|| value[plus_plus_len] != '+'
|| (value[plus_plus_len + 1] != '+' && value[plus_plus_len + 1] != '=')) {
goto send;
}
if(a) {
v = g_hash_table_lookup(a->values, host);
}
if(v) {
int_value = char2score(v->current);
}
if (value[plus_plus_len + 1] != '+') {
const char *offset_s = value + (plus_plus_len + 2);
offset = char2score(offset_s);
}
int_value += offset;
if (int_value > INFINITY) {
int_value = INFINITY;
}
crm_info("Expanded %s=%s to %d", attr, value, int_value);
crm_xml_add_int(xml, F_ATTRD_VALUE, int_value);
}
send:
if(peer_writer == NULL && election_state(writer) != election_in_progress) {
crm_info("Starting an election to determine the writer");
election_vote(writer);
}
- crm_info("Broadcasting %s[%s] = %s%s", attr, host, value, election_state(writer) == election_won?" (writer)":"");
+ crm_debug("Broadcasting %s[%s] = %s%s", attr, host, value, election_state(writer) == election_won?" (writer)":"");
broadcast = TRUE;
free(key);
free(set);
free(host);
} else if(safe_str_eq(op, "refresh")) {
crm_notice("Updating all attributes");
write_attributes(TRUE, FALSE);
}
if(broadcast) {
send_attrd_message(NULL, xml);
}
}
void
attrd_peer_message(crm_node_t *peer, xmlNode *xml)
{
int peer_state = 0;
const char *v = crm_element_value(xml, F_ATTRD_VERSION);
const char *op = crm_element_value(xml, F_ATTRD_TASK);
const char *election_op = crm_element_value(xml, F_CRM_TASK);
if(election_op) {
enum election_result rc = 0;
crm_xml_add(xml, F_CRM_HOST_FROM, peer->uname);
rc = election_count_vote(writer, xml, TRUE);
switch(rc) {
case election_start:
free(peer_writer);
peer_writer = NULL;
election_vote(writer);
break;
case election_lost:
free(peer_writer);
peer_writer = strdup(peer->uname);
break;
default:
election_check(writer);
break;
}
return;
} else if(v == NULL) {
/* From the non-atomic version */
if(safe_str_eq(op, "update")) {
const char *name = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
crm_trace("Compatibility update of %s from %s", name, peer->uname);
attrd_peer_update(peer, xml, FALSE);
} else if(safe_str_eq(op, "flush")) {
const char *name = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
attribute_t *a = g_hash_table_lookup(attributes, name);
if(a) {
crm_trace("Compatibility write-out of %s for %s from %s", a->id, op, peer->uname);
write_or_elect_attribute(a);
}
} else if(safe_str_eq(op, "refresh")) {
GHashTableIter aIter;
attribute_t *a = NULL;
g_hash_table_iter_init(&aIter, attributes);
while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
crm_trace("Compatibility write-out of %s for %s from %s", a->id, op, peer->uname);
write_or_elect_attribute(a);
}
}
}
crm_element_value_int(xml, F_ATTRD_WRITER, &peer_state);
if(election_state(writer) == election_won
&& peer_state == election_won
&& safe_str_neq(peer->uname, attrd_cluster->uname)) {
crm_notice("Detected another attribute writer: %s", peer->uname);
election_vote(writer);
} else if(peer_state == election_won) {
if(peer_writer == NULL) {
peer_writer = strdup(peer->uname);
crm_notice("Recorded attribute writer: %s", peer->uname);
} else if(safe_str_neq(peer->uname, peer_writer)) {
crm_notice("Recorded new attribute writer: %s (was %s)", peer->uname, peer_writer);
free(peer_writer);
peer_writer = strdup(peer->uname);
}
}
if(safe_str_eq(op, "update")) {
attrd_peer_update(peer, xml, FALSE);
} else if(safe_str_eq(op, "sync")) {
attrd_peer_sync(peer, xml);
} else if(safe_str_eq(op, "peer-remove")) {
const char *host = crm_element_value(xml, F_ATTRD_HOST);
attrd_peer_remove(host, peer->uname);
} else if(safe_str_eq(op, "sync-response")
&& safe_str_neq(peer->uname, attrd_cluster->uname)) {
xmlNode *child = NULL;
crm_notice("Processing %s from %s", op, peer->uname);
for (child = __xml_first_child(xml); child != NULL; child = __xml_next(child)) {
attrd_peer_update(peer, child, TRUE);
}
}
}
void
attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
{
GHashTableIter aIter;
GHashTableIter vIter;
attribute_t *a = NULL;
attribute_value_t *v = NULL;
xmlNode *sync = create_xml_node(NULL, __FUNCTION__);
crm_xml_add(sync, F_ATTRD_TASK, "sync-response");
g_hash_table_iter_init(&aIter, attributes);
while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
g_hash_table_iter_init(&vIter, a->values);
while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone");
build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, v->nodename, v->nodeid, v->current);
}
}
crm_debug("Syncing values to %s", peer?peer->uname:"everyone");
send_attrd_message(peer, sync);
free_xml(sync);
}
void
attrd_peer_remove(const char *host, const char *source)
{
attribute_t *a = NULL;
GHashTableIter aIter;
crm_notice("Removing all %s attributes for %s", host, source);
if(host == NULL) {
return;
}
g_hash_table_iter_init(&aIter, attributes);
while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
if(g_hash_table_remove(a->values, host)) {
crm_debug("Removed %s[%s] for %s", a->id, host, source);
}
}
/* if this matches a remote peer, it will be removed from the cache */
crm_remote_peer_cache_remove(host);
}
void
attrd_peer_update(crm_node_t *peer, xmlNode *xml, bool filter)
{
bool changed = FALSE;
attribute_value_t *v = NULL;
const char *host = crm_element_value(xml, F_ATTRD_HOST);
const char *attr = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
const char *value = crm_element_value(xml, F_ATTRD_VALUE);
attribute_t *a = g_hash_table_lookup(attributes, attr);
if(a == NULL) {
a = create_attribute(xml);
}
v = g_hash_table_lookup(a->values, host);
if(v == NULL) {
- crm_trace("Setting %s[%s] to %s from %s", attr, host, value, peer->uname);
v = calloc(1, sizeof(attribute_value_t));
- if(value) {
- v->current = strdup(value);
- }
v->nodename = strdup(host);
crm_element_value_int(xml, F_ATTRD_IS_REMOTE, &v->is_remote);
g_hash_table_replace(a->values, v->nodename, v);
if (v->is_remote == TRUE) {
crm_remote_peer_cache_add(host);
}
+ }
- changed = TRUE;
-
- } else if(filter
+ if(filter
&& safe_str_neq(v->current, value)
&& safe_str_eq(host, attrd_cluster->uname)) {
xmlNode *sync = create_xml_node(NULL, __FUNCTION__);
crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s",
a->id, host, v->current, value, peer->uname);
crm_xml_add(sync, F_ATTRD_TASK, "sync-response");
v = g_hash_table_lookup(a->values, host);
build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, v->nodename, v->nodeid, v->current);
crm_xml_add_int(sync, F_ATTRD_WRITER, election_state(writer));
send_attrd_message(peer, sync);
free_xml(sync);
} else if(safe_str_neq(v->current, value)) {
crm_info("Setting %s[%s]: %s -> %s from %s", attr, host, v->current, value, peer->uname);
free(v->current);
if(value) {
v->current = strdup(value);
} else {
v->current = NULL;
}
changed = TRUE;
} else {
crm_trace("Unchanged %s[%s] from %s is %s", attr, host, peer->uname, value);
}
a->changed |= changed;
/* this only involves cluster nodes. */
if(v->nodeid == 0 && (v->is_remote == FALSE)) {
if(crm_element_value_int(xml, F_ATTRD_HOST_ID, (int*)&v->nodeid) == 0) {
/* Create the name/id association */
crm_node_t *peer = crm_get_peer(v->nodeid, host);
crm_trace("We know %s's node id now: %s", peer->uname, peer->uuid);
if(election_state(writer) == election_won) {
write_attributes(FALSE, TRUE);
return;
}
}
}
if(changed) {
if(a->timer) {
crm_trace("Delayed write out (%dms) for %s", a->timeout_ms, a->id);
mainloop_timer_start(a->timer);
} else {
write_or_elect_attribute(a);
}
}
}
void
write_or_elect_attribute(attribute_t *a)
{
enum election_result rc = election_state(writer);
if(rc == election_won) {
write_attribute(a);
} else if(rc == election_in_progress) {
crm_trace("Election in progress to determine who will write out %s", a->id);
} else if(peer_writer == NULL) {
crm_info("Starting an election to determine who will write out %s", a->id);
election_vote(writer);
} else {
crm_trace("%s will write out %s, we are in state %d", peer_writer, a->id, rc);
}
}
gboolean
attrd_election_cb(gpointer user_data)
{
crm_trace("Election complete");
free(peer_writer);
peer_writer = strdup(attrd_cluster->uname);
/* Update the peers after an election */
attrd_peer_sync(NULL, NULL);
/* Update the CIB after an election */
write_attributes(TRUE, FALSE);
return FALSE;
}
void
attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
{
if(election_state(writer) == election_won
&& kind == crm_status_nstate
&& safe_str_eq(peer->state, CRM_NODE_MEMBER)) {
attrd_peer_sync(peer, NULL);
} else if(kind == crm_status_nstate
&& safe_str_neq(peer->state, CRM_NODE_MEMBER)) {
attrd_peer_remove(peer->uname, __FUNCTION__);
if(peer_writer && safe_str_eq(peer->uname, peer_writer)) {
free(peer_writer);
peer_writer = NULL;
crm_notice("Lost attribute writer %s", peer->uname);
}
} else if(kind == crm_status_processes) {
if(is_set(peer->processes, crm_proc_cpg)) {
crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0);
} else {
crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0);
}
}
}
static void
attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
int level = LOG_ERR;
GHashTableIter iter;
const char *peer = NULL;
attribute_value_t *v = NULL;
char *name = user_data;
attribute_t *a = g_hash_table_lookup(attributes, name);
if(a == NULL) {
crm_info("Attribute %s no longer exists", name);
goto done;
}
a->update = 0;
if (rc == pcmk_ok && call_id < 0) {
rc = call_id;
}
switch (rc) {
case pcmk_ok:
level = LOG_INFO;
last_cib_op_done = call_id;
break;
case -pcmk_err_diff_failed: /* When an attr changes while the CIB is syncing */
case -ETIME: /* When an attr changes while there is a DC election */
case -ENXIO: /* When an attr changes while the CIB is syncing a
* newer config from a node that just came up
*/
level = LOG_WARNING;
break;
}
do_crm_log(level, "Update %d for %s: %s (%d)", call_id, name, pcmk_strerror(rc), rc);
g_hash_table_iter_init(&iter, a->values);
while (g_hash_table_iter_next(&iter, (gpointer *) & peer, (gpointer *) & v)) {
crm_notice("Update %d for %s[%s]=%s: %s (%d)", call_id, a->id, peer, v->requested, pcmk_strerror(rc), rc);
if(rc == pcmk_ok) {
free(v->stored);
v->stored = v->requested;
v->requested = NULL;
} else {
free(v->requested);
v->requested = NULL;
a->changed = TRUE; /* Attempt write out again */
}
}
done:
free(name);
if(a && a->changed && election_state(writer) == election_won) {
write_attribute(a);
}
}
void
write_attributes(bool all, bool peer_discovered)
{
GHashTableIter iter;
attribute_t *a = NULL;
g_hash_table_iter_init(&iter, attributes);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) {
if (peer_discovered && a->unknown_peer_uuids) {
/* a new peer uuid has been discovered, try writing this attribute again. */
a->changed = TRUE;
}
if(all || a->changed) {
write_attribute(a);
} else {
crm_debug("Skipping unchanged attribute %s", a->id);
}
}
}
static void
build_update_element(xmlNode *parent, attribute_t *a, const char *nodeid, const char *value)
{
char *set = NULL;
char *uuid = NULL;
xmlNode *xml_obj = NULL;
if(a->set) {
set = g_strdup(a->set);
} else {
set = g_strdup_printf("%s-%s", XML_CIB_TAG_STATUS, nodeid);
}
if(a->uuid) {
uuid = g_strdup(a->uuid);
} else {
int lpc;
uuid = g_strdup_printf("%s-%s", set, a->id);
/* Minimal attempt at sanitizing automatic IDs */
for (lpc = 0; uuid[lpc] != 0; lpc++) {
switch (uuid[lpc]) {
case ':':
uuid[lpc] = '.';
}
}
}
xml_obj = create_xml_node(parent, XML_CIB_TAG_STATE);
crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);
xml_obj = create_xml_node(xml_obj, XML_TAG_TRANSIENT_NODEATTRS);
crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);
xml_obj = create_xml_node(xml_obj, XML_TAG_ATTR_SETS);
crm_xml_add(xml_obj, XML_ATTR_ID, set);
xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
crm_xml_add(xml_obj, XML_ATTR_ID, uuid);
crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, a->id);
if(value) {
crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, value);
} else {
crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, "");
crm_xml_add(xml_obj, "__delete__", XML_NVPAIR_ATTR_VALUE);
}
g_free(uuid);
g_free(set);
}
void
write_attribute(attribute_t *a)
{
int updates = 0;
xmlNode *xml_top = NULL;
attribute_value_t *v = NULL;
GHashTableIter iter;
enum cib_call_options flags = cib_quorum_override;
if (a == NULL) {
return;
} else if (the_cib == NULL) {
crm_info("Write out of '%s' delayed: cib not connected", a->id);
return;
} else if(a->update && a->update < last_cib_op_done) {
crm_info("Write out of '%s' continuing: update %d considered lost", a->id, a->update);
} else if(a->update) {
crm_info("Write out of '%s' delayed: update %d in progress", a->id, a->update);
return;
} else if(mainloop_timer_running(a->timer)) {
crm_info("Write out of '%s' delayed: timer is running", a->id);
return;
}
a->changed = FALSE;
a->unknown_peer_uuids = FALSE;
xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
g_hash_table_iter_init(&iter, a->values);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & v)) {
crm_node_t *peer = crm_get_peer_full(v->nodeid, v->nodename, CRM_GET_PEER_REMOTE|CRM_GET_PEER_CLUSTER);
if(peer && peer->id && v->nodeid == 0) {
crm_trace("Updating value's nodeid");
v->nodeid = peer->id;
}
if (peer == NULL) {
/* If the user is trying to set an attribute on an unknown peer, ignore it. */
crm_notice("Update error (peer not found): %s[%s]=%s failed (host=%p)", v->nodename, a->id, v->current, peer);
} else if (peer->uuid == NULL) {
/* peer is found, but we don't know the uuid yet. Wait until we discover a new uuid before attempting to write */
a->unknown_peer_uuids = FALSE;
crm_notice("Update error (unknown peer uuid, retry will be attempted once uuid is discovered): %s[%s]=%s failed (host=%p)", v->nodename, a->id, v->current, peer);
} else {
crm_debug("Update: %s[%s]=%s (%s %u %u %s)", v->nodename, a->id, v->current, peer->uuid, peer->id, v->nodeid, peer->uname);
build_update_element(xml_top, a, peer->uuid, v->current);
updates++;
free(v->requested);
v->requested = NULL;
if(v->current) {
v->requested = strdup(v->current);
} else {
/* Older versions don't know about the cib_mixed_update flag
* Make sure it goes to the local cib which does
*/
flags |= cib_mixed_update|cib_scope_local;
}
}
}
if(updates) {
crm_log_xml_trace(xml_top, __FUNCTION__);
a->update = cib_internal_op(the_cib, CIB_OP_MODIFY, NULL, XML_CIB_TAG_STATUS, xml_top, NULL,
flags, a->user);
crm_notice("Sent update %d with %d changes for %s, id=%s, set=%s",
a->update, updates, a->id, a->uuid ? a->uuid : "<n/a>", a->set);
the_cib->cmds->register_callback(
the_cib, a->update, 120, FALSE, strdup(a->id), "attrd_cib_callback", attrd_cib_callback);
}
free_xml(xml_top);
}
diff --git a/cib/io.c b/cib/io.c
index 0db8204839..17e5a9e37e 100644
--- a/cib/io.c
+++ b/cib/io.c
@@ -1,823 +1,828 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <dirent.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <crm/crm.h>
#include <cibio.h>
#include <crm/cib.h>
#include <crm/common/util.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/util.h>
#include <crm/cluster.h>
#define CIB_SERIES "cib"
#define CIB_SERIES_MAX 100
#define CIB_SERIES_BZIP FALSE /* Must be false due to the way archived
* copies are created - ie. with calls to
* link()
*/
extern const char *cib_root;
#define CIB_WRITE_PARANOIA 0
const char *local_resource_path[] = {
XML_CIB_TAG_STATUS,
};
const char *resource_path[] = {
XML_CIB_TAG_RESOURCES,
};
const char *node_path[] = {
XML_CIB_TAG_NODES,
};
const char *constraint_path[] = {
XML_CIB_TAG_CONSTRAINTS,
};
crm_trigger_t *cib_writer = NULL;
gboolean initialized = FALSE;
xmlNode *node_search = NULL;
xmlNode *resource_search = NULL;
xmlNode *constraint_search = NULL;
xmlNode *status_search = NULL;
extern int cib_status;
int set_connected_peers(xmlNode * xml_obj);
void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data);
int write_cib_contents(gpointer p);
extern void cib_cleanup(void);
static gboolean
validate_cib_digest(xmlNode * local_cib, const char *sigfile)
{
char *digest = NULL;
char *expected = NULL;
gboolean passed = FALSE;
FILE *expected_strm = NULL;
int start = 0, length = 0, read_len = 0;
CRM_ASSERT(sigfile != NULL);
expected_strm = fopen(sigfile, "r");
if (expected_strm == NULL && errno == ENOENT) {
crm_warn("No on-disk digest present");
return TRUE;
} else if (expected_strm == NULL) {
crm_perror(LOG_ERR, "Could not open signature file %s for reading", sigfile);
goto bail;
}
if (local_cib != NULL) {
digest = calculate_on_disk_digest(local_cib);
}
start = ftell(expected_strm);
fseek(expected_strm, 0L, SEEK_END);
length = ftell(expected_strm);
fseek(expected_strm, 0L, start);
CRM_ASSERT(length >= 0);
CRM_ASSERT(start == ftell(expected_strm));
if (length > 0) {
crm_trace("Reading %d bytes from file", length);
expected = calloc(1, (length + 1));
read_len = fread(expected, 1, length, expected_strm); /* Coverity: False positive */
CRM_ASSERT(read_len == length);
}
fclose(expected_strm);
bail:
if (expected == NULL) {
crm_err("On-disk digest is empty");
} else if (safe_str_eq(expected, digest)) {
crm_trace("Digest comparision passed: %s", digest);
passed = TRUE;
} else {
crm_err("Digest comparision failed: expected %s (%s), calculated %s",
expected, sigfile, digest);
}
free(digest);
free(expected);
return passed;
}
static int
write_cib_digest(xmlNode * local_cib, const char *digest_file, int fd, char *digest)
{
int rc = 0;
char *local_digest = NULL;
FILE *digest_strm = fdopen(fd, "w");
if (digest_strm == NULL) {
crm_perror(LOG_ERR, "Cannot open signature file %s for writing", digest_file);
return -1;
}
if (digest == NULL) {
local_digest = calculate_on_disk_digest(local_cib);
CRM_ASSERT(digest != NULL);
digest = local_digest;
}
rc = fprintf(digest_strm, "%s", digest);
if (rc < 0) {
crm_perror(LOG_ERR, "Cannot write to signature file %s", digest_file);
}
CRM_ASSERT(digest_strm != NULL);
if (fflush(digest_strm) != 0) {
crm_perror(LOG_ERR, "Couldnt flush the contents of %s", digest_file);
rc = -1;
}
if (fsync(fileno(digest_strm)) < 0) {
crm_perror(LOG_ERR, "Couldnt sync the contents of %s", digest_file);
rc = -1;
}
fclose(digest_strm);
free(local_digest);
return rc;
}
static gboolean
validate_on_disk_cib(const char *filename, xmlNode ** on_disk_cib)
{
int s_res = -1;
struct stat buf;
gboolean passed = TRUE;
xmlNode *root = NULL;
CRM_ASSERT(filename != NULL);
s_res = stat(filename, &buf);
if (s_res == 0) {
char *sigfile = NULL;
size_t fnsize;
+ if (buf.st_size == 0) {
+ crm_warn("Cluster configuration file %s is corrupt: size is zero", filename);
+ return TRUE;
+ }
+
crm_trace("Reading cluster configuration from: %s", filename);
root = filename2xml(filename);
fnsize = strlen(filename) + 5;
sigfile = calloc(1, fnsize);
snprintf(sigfile, fnsize, "%s.sig", filename);
if (validate_cib_digest(root, sigfile) == FALSE) {
passed = FALSE;
}
free(sigfile);
}
if (on_disk_cib != NULL) {
*on_disk_cib = root;
} else {
free_xml(root);
}
return passed;
}
static int
cib_rename(const char *old, const char *new)
{
int rc = 0;
int automatic_fd = 0;
char *automatic = NULL;
if (new == NULL) {
umask(S_IWGRP | S_IWOTH | S_IROTH);
automatic = g_strdup_printf("%s/cib.auto.XXXXXX", cib_root);
automatic_fd = mkstemp(automatic);
new = automatic;
crm_err("Archiving corrupt or unusable file %s as %s", old, automatic);
}
rc = rename(old, new);
if (rc < 0) {
crm_perror(LOG_ERR, "Couldn't rename %s as %s - Disabling disk writes and continuing", old,
new);
cib_writes_enabled = FALSE;
}
if (automatic_fd > 0) {
close(automatic_fd);
}
free(automatic);
return rc;
}
/*
* It is the callers responsibility to free the output of this function
*/
static xmlNode *
retrieveCib(const char *filename, const char *sigfile, gboolean archive_invalid)
{
struct stat buf;
xmlNode *root = NULL;
crm_info("Reading cluster configuration from: %s (digest: %s)", filename, sigfile);
if (stat(filename, &buf) != 0) {
crm_warn("Cluster configuration not found: %s", filename);
return NULL;
}
root = filename2xml(filename);
if (root == NULL) {
crm_err("%s exists but does NOT contain valid XML. ", filename);
crm_warn("Continuing but %s will NOT used.", filename);
} else if (validate_cib_digest(root, sigfile) == FALSE) {
crm_err("Checksum of %s failed! Configuration contents ignored!", filename);
crm_err("Usually this is caused by manual changes, "
"please refer to http://clusterlabs.org/wiki/FAQ#cib_changes_detected");
crm_warn("Continuing but %s will NOT used.", filename);
free_xml(root);
root = NULL;
if (archive_invalid) {
/* Archive the original files so the contents are not lost */
cib_rename(filename, NULL);
cib_rename(sigfile, NULL);
}
}
return root;
}
static int cib_archive_filter(const struct dirent * a)
{
int rc = 0;
/* Looking for regular files (d_type = 8) starting with 'cib-' and not ending in .sig */
struct stat s;
char *a_path = g_strdup_printf("%s/%s", cib_root, a->d_name);
if(stat(a_path, &s) != 0) {
rc = errno;
crm_trace("%s - stat failed: %s (%d)", a->d_name, pcmk_strerror(rc), rc);
rc = 0;
} else if ((s.st_mode & S_IFREG) != S_IFREG) {
crm_trace("%s - wrong type (%d)", a->d_name, a->d_type);
} else if(strstr(a->d_name, "cib-") != a->d_name) {
crm_trace("%s - wrong prefix", a->d_name);
} else if(strstr(a->d_name, ".sig") != NULL) {
crm_trace("%s - wrong suffix", a->d_name);
} else {
crm_debug("%s - candidate", a->d_name);
rc = 1;
}
free(a_path);
return rc;
}
static int cib_archive_sort(const struct dirent ** a, const struct dirent **b)
{
/* Order by creation date - most recently created file first */
int rc = 0;
struct stat buf;
time_t a_age = 0;
time_t b_age = 0;
char *a_path = g_strdup_printf("%s/%s", cib_root, a[0]->d_name);
char *b_path = g_strdup_printf("%s/%s", cib_root, b[0]->d_name);
if(stat(a_path, &buf) == 0) {
a_age = buf.st_ctime;
}
if(stat(b_path, &buf) == 0) {
b_age = buf.st_ctime;
}
free(a_path);
free(b_path);
if(a_age > b_age) {
rc = 1;
} else if(a_age < b_age) {
rc = -1;
}
crm_trace("%s (%u) vs. %s (%u) : %d", a[0]->d_name, a_age, b[0]->d_name, b_age, rc);
return rc;
}
xmlNode *
readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
{
struct dirent **namelist = NULL;
int lpc = 0;
char *sigfile = NULL;
char *filename = NULL;
const char *name = NULL;
const char *value = NULL;
const char *validation = NULL;
const char *use_valgrind = getenv("PCMK_valgrind_enabled");
xmlNode *root = NULL;
xmlNode *status = NULL;
if (!crm_is_writable(dir, file, CRM_DAEMON_USER, NULL, FALSE)) {
cib_status = -EACCES;
return NULL;
}
filename = crm_concat(dir, file, '/');
sigfile = crm_concat(filename, "sig", '.');
cib_status = pcmk_ok;
root = retrieveCib(filename, sigfile, TRUE);
free(filename);
free(sigfile);
if (root == NULL) {
crm_warn("Primary configuration corrupt or unusable, trying backups in %s", cib_root);
lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort);
if (lpc < 0) {
crm_perror(LOG_NOTICE, "scandir(%s) failed", cib_root);
}
}
while (root == NULL && lpc > 1) {
crm_debug("Testing %d candidates", lpc);
lpc--;
filename = g_strdup_printf("%s/%s", cib_root, namelist[lpc]->d_name);
sigfile = crm_concat(filename, "sig", '.');
root = retrieveCib(filename, sigfile, FALSE);
if(root) {
crm_notice("Continuing with last valid configuration archive: %s", filename);
}
free(namelist[lpc]);
free(filename);
free(sigfile);
}
free(namelist);
if (root == NULL) {
root = createEmptyCib();
crm_xml_add(root, XML_ATTR_GENERATION, "0");
crm_xml_add(root, XML_ATTR_NUMUPDATES, "0");
crm_xml_add(root, XML_ATTR_GENERATION_ADMIN, "0");
crm_xml_add(root, XML_ATTR_VALIDATION, LATEST_SCHEMA_VERSION);
crm_warn("Continuing with an empty configuration.");
}
if (cib_writes_enabled && use_valgrind) {
if (crm_is_true(use_valgrind) || strstr(use_valgrind, "cib")) {
cib_writes_enabled = FALSE;
crm_err("*********************************************************");
crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
crm_err("*********************************************************");
}
}
status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
if (discard_status && status != NULL) {
/* strip out the status section if there is one */
free_xml(status);
status = NULL;
}
if (status == NULL) {
create_xml_node(root, XML_CIB_TAG_STATUS);
}
/* Do this before DTD validation happens */
/* fill in some defaults */
name = XML_ATTR_GENERATION_ADMIN;
value = crm_element_value(root, name);
if (value == NULL) {
crm_warn("No value for %s was specified in the configuration.", name);
crm_warn("The reccomended course of action is to shutdown,"
" run crm_verify and fix any errors it reports.");
crm_warn("We will default to zero and continue but may get"
" confused about which configuration to use if"
" multiple nodes are powered up at the same time.");
crm_xml_add_int(root, name, 0);
}
name = XML_ATTR_GENERATION;
value = crm_element_value(root, name);
if (value == NULL) {
crm_xml_add_int(root, name, 0);
}
name = XML_ATTR_NUMUPDATES;
value = crm_element_value(root, name);
if (value == NULL) {
crm_xml_add_int(root, name, 0);
}
/* unset these and require the DC/CCM to update as needed */
xml_remove_prop(root, XML_ATTR_DC_UUID);
if (discard_status) {
crm_log_xml_trace(root, "[on-disk]");
}
validation = crm_element_value(root, XML_ATTR_VALIDATION);
if (validate_xml(root, NULL, TRUE) == FALSE) {
crm_err("CIB does not validate with %s", crm_str(validation));
cib_status = -pcmk_err_dtd_validation;
} else if (validation == NULL) {
int version = 0;
update_validation(&root, &version, FALSE, FALSE);
if (version > 0) {
crm_notice("Enabling %s validation on"
" the existing (sane) configuration", get_schema_name(version));
} else {
crm_err("CIB does not validate with any known DTD or schema");
cib_status = -pcmk_err_dtd_validation;
}
}
return root;
}
/*
* The caller should never free the return value
*/
xmlNode *
get_the_CIB(void)
{
return the_cib;
}
gboolean
uninitializeCib(void)
{
xmlNode *tmp_cib = the_cib;
if (tmp_cib == NULL) {
crm_debug("The CIB has already been deallocated.");
return FALSE;
}
initialized = FALSE;
the_cib = NULL;
node_search = NULL;
resource_search = NULL;
constraint_search = NULL;
status_search = NULL;
crm_debug("Deallocating the CIB.");
free_xml(tmp_cib);
crm_debug("The CIB has been deallocated.");
return TRUE;
}
/*
* This method will not free the old CIB pointer or the new one.
* We rely on the caller to have saved a pointer to the old CIB
* and to free the old/bad one depending on what is appropriate.
*/
gboolean
initializeCib(xmlNode * new_cib)
{
if (new_cib == NULL) {
return FALSE;
}
the_cib = new_cib;
initialized = TRUE;
return TRUE;
}
static void
sync_directory(const char *name)
{
int fd = 0;
DIR *directory = NULL;
directory = opendir(name);
if (directory == NULL) {
crm_perror(LOG_ERR, "Could not open %s for syncing", name);
return;
}
fd = dirfd(directory);
if (fd < 0) {
crm_perror(LOG_ERR, "Could not obtain file descriptor for %s", name);
} else if (fsync(fd) < 0) {
crm_perror(LOG_ERR, "Could not sync %s", name);
}
if (closedir(directory) < 0) {
crm_perror(LOG_ERR, "Could not close %s after fsync", name);
}
}
/*
* This method will free the old CIB pointer on success and the new one
* on failure.
*/
int
activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op)
{
xmlNode *saved_cib = the_cib;
CRM_ASSERT(new_cib != saved_cib);
if (initializeCib(new_cib) == FALSE) {
free_xml(new_cib);
crm_err("Ignoring invalid or NULL CIB");
if (saved_cib != NULL) {
crm_warn("Reverting to last known CIB");
if (initializeCib(saved_cib) == FALSE) {
/* oh we are so dead */
crm_crit("Couldn't re-initialize the old CIB!");
exit(1);
}
} else {
crm_crit("Could not write out new CIB and no saved" " version to revert to");
}
return -ENODATA;
}
free_xml(saved_cib);
if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) {
crm_debug("Triggering CIB write for %s op", op);
mainloop_set_trigger(cib_writer);
}
return pcmk_ok;
}
static void
cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
{
if (signo) {
crm_notice("Disk write process terminated with signal %d (pid=%d, core=%d)", signo, pid,
core);
} else {
do_crm_log(exitcode == 0 ? LOG_TRACE : LOG_ERR, "Disk write process exited (pid=%d, rc=%d)",
pid, exitcode);
}
if (exitcode != 0 && cib_writes_enabled) {
crm_err("Disabling disk writes after write failure");
cib_writes_enabled = FALSE;
}
mainloop_trigger_complete(cib_writer);
}
int
write_cib_contents(gpointer p)
{
int exit_rc = pcmk_ok;
char *digest = NULL;
xmlNode *cib_status_root = NULL;
xmlNode *cib_local = NULL;
xmlNode *cib_tmp = NULL;
int tmp_cib_fd = 0;
int tmp_digest_fd = 0;
char *tmp_cib = NULL;
char *tmp_digest = NULL;
char *digest_file = NULL;
char *primary_file = NULL;
char *backup_file = NULL;
char *backup_digest = NULL;
const char *epoch = NULL;
const char *admin_epoch = NULL;
if (p) {
/* Synchronous write out */
cib_local = copy_xml(p);
} else {
int pid = 0;
int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0);
/* Turn it off before the fork() to avoid:
* - 2 processes writing to the same shared mem
* - the child needing to disable it
* (which would close it from underneath the parent)
* This way, the shared mem files are already closed
*/
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);
pid = fork();
if (pid < 0) {
crm_perror(LOG_ERR, "Disabling disk writes after fork failure");
cib_writes_enabled = FALSE;
return FALSE;
}
if (pid) {
/* Parent */
mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete);
if (bb_state == QB_LOG_STATE_ENABLED) {
/* Re-enable now that it it safe */
qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
}
return -1; /* -1 means 'still work to do' */
}
/* A-synchronous write out after a fork() */
/* In theory we can scribble on "the_cib" here and not affect the parent
* But lets be safe anyway
*/
cib_local = copy_xml(the_cib);
}
epoch = crm_element_value(cib_local, XML_ATTR_GENERATION);
admin_epoch = crm_element_value(cib_local, XML_ATTR_GENERATION_ADMIN);
primary_file = crm_concat(cib_root, "cib.xml", '/');
digest_file = crm_concat(primary_file, "sig", '.');
/* Always write out with num_updates=0 */
crm_xml_add(cib_local, XML_ATTR_NUMUPDATES, "0");
/* check the admin didnt modify it underneath us */
if (validate_on_disk_cib(primary_file, NULL) == FALSE) {
crm_err("%s was manually modified while the cluster was active!", primary_file);
exit_rc = pcmk_err_cib_modified;
goto cleanup;
} else {
int rc = 0;
int seq = get_last_sequence(cib_root, CIB_SERIES);
backup_file = generate_series_filename(cib_root, CIB_SERIES, seq, CIB_SERIES_BZIP);
backup_digest = crm_concat(backup_file, "sig", '.');
unlink(backup_file);
unlink(backup_digest);
rc = link(primary_file, backup_file);
if (rc < 0) {
rc = errno;
switch(rc) {
case ENOENT:
/* No file to back up */
goto writeout;
break;
default:
exit_rc = pcmk_err_cib_backup;
crm_err("Cannot link %s to %s: %s (%d)", primary_file, backup_file, pcmk_strerror(rc), rc);
}
goto cleanup;
}
rc = link(digest_file, backup_digest);
if (rc < 0 && errno != ENOENT) {
exit_rc = pcmk_err_cib_backup;
crm_perror(LOG_ERR, "Cannot link %s to %s", digest_file, backup_digest);
goto cleanup;
}
write_last_sequence(cib_root, CIB_SERIES, seq + 1, CIB_SERIES_MAX);
sync_directory(cib_root);
crm_info("Archived previous version as %s", backup_file);
}
writeout:
/* Given that we discard the status section on startup
* there is no point writing it out in the first place
* since users just get confused by it
*
* So delete the status section before we write it out
*/
crm_debug("Writing CIB to disk");
if (p == NULL) {
cib_status_root = find_xml_node(cib_local, XML_CIB_TAG_STATUS, TRUE);
CRM_LOG_ASSERT(cib_status_root != NULL);
if (cib_status_root != NULL) {
free_xml(cib_status_root);
}
}
tmp_cib = g_strdup_printf("%s/cib.XXXXXX", cib_root);
tmp_digest = g_strdup_printf("%s/cib.XXXXXX", cib_root);
umask(S_IWGRP | S_IWOTH | S_IROTH);
tmp_cib_fd = mkstemp(tmp_cib);
if (tmp_cib_fd < 0 || write_xml_fd(cib_local, tmp_cib, tmp_cib_fd, FALSE) <= 0) {
crm_err("Changes couldn't be written to %s", tmp_cib);
exit_rc = pcmk_err_cib_save;
goto cleanup;
}
/* Must calculate the digest after writing as write_xml_file() updates the last-written field */
digest = calculate_on_disk_digest(cib_local);
crm_info("Wrote version %s.%s.0 of the CIB to disk (digest: %s)",
admin_epoch ? admin_epoch : "0", epoch ? epoch : "0", digest);
tmp_digest_fd = mkstemp(tmp_digest);
if (tmp_digest_fd < 0 || write_cib_digest(cib_local, tmp_digest, tmp_digest_fd, digest) <= 0) {
crm_err("Digest couldn't be written to %s", tmp_digest);
exit_rc = pcmk_err_cib_save;
goto cleanup;
}
crm_debug("Wrote digest %s to disk", digest);
cib_tmp = retrieveCib(tmp_cib, tmp_digest, FALSE);
CRM_ASSERT(cib_tmp != NULL);
sync_directory(cib_root);
crm_debug("Activating %s", tmp_cib);
cib_rename(tmp_cib, primary_file);
cib_rename(tmp_digest, digest_file);
sync_directory(cib_root);
cleanup:
free(backup_digest);
free(primary_file);
free(backup_file);
free(digest_file);
free(digest);
free(tmp_digest);
free(tmp_cib);
free_xml(cib_tmp);
free_xml(cib_local);
if (p == NULL) {
/* exit() could potentially affect the parent by closing things it shouldn't
* Use _exit instead
*/
_exit(exit_rc);
}
return exit_rc;
}
void
GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data)
{
int *active = user_data;
if (safe_str_eq(value, ONLINESTATUS)) {
(*active)++;
} else if (safe_str_eq(value, JOINSTATUS)) {
(*active)++;
}
}
diff --git a/lib/common/ipc.c b/lib/common/ipc.c
index b951218a49..310337e87b 100644
--- a/lib/common/ipc.c
+++ b/lib/common/ipc.c
@@ -1,1237 +1,1237 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <grp.h>
#include <errno.h>
#include <fcntl.h>
#include <bzlib.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipcs.h>
#define PCMK_IPC_VERSION 1
struct crm_ipc_response_header {
struct qb_ipc_response_header qb;
uint32_t size_uncompressed;
uint32_t size_compressed;
uint32_t flags;
uint8_t version; /* Protect against version changes for anyone that might bother to statically link us */
};
static int hdr_offset = 0;
static int ipc_buffer_max = 0;
static unsigned int pick_ipc_buffer(int max);
static inline void
crm_ipc_init(void)
{
if (hdr_offset == 0) {
hdr_offset = sizeof(struct crm_ipc_response_header);
}
if (ipc_buffer_max == 0) {
ipc_buffer_max = pick_ipc_buffer(0);
}
}
int
crm_ipc_default_buffer_size(void)
{
return pick_ipc_buffer(0);
}
static char *
generateReference(const char *custom1, const char *custom2)
{
static uint ref_counter = 0;
const char *local_cust1 = custom1;
const char *local_cust2 = custom2;
int reference_len = 4;
char *since_epoch = NULL;
reference_len += 20; /* too big */
reference_len += 40; /* too big */
if (local_cust1 == NULL) {
local_cust1 = "_empty_";
}
reference_len += strlen(local_cust1);
if (local_cust2 == NULL) {
local_cust2 = "_empty_";
}
reference_len += strlen(local_cust2);
since_epoch = calloc(1, reference_len);
if (since_epoch != NULL) {
sprintf(since_epoch, "%s-%s-%ld-%u",
local_cust1, local_cust2, (unsigned long)time(NULL), ref_counter++);
}
return since_epoch;
}
xmlNode *
create_request_adv(const char *task, xmlNode * msg_data,
const char *host_to, const char *sys_to,
const char *sys_from, const char *uuid_from, const char *origin)
{
char *true_from = NULL;
xmlNode *request = NULL;
char *reference = generateReference(task, sys_from);
if (uuid_from != NULL) {
true_from = generate_hash_key(sys_from, uuid_from);
} else if (sys_from != NULL) {
true_from = strdup(sys_from);
} else {
crm_err("No sys from specified");
}
/* host_from will get set for us if necessary by CRMd when routed */
request = create_xml_node(NULL, __FUNCTION__);
crm_xml_add(request, F_CRM_ORIGIN, origin);
crm_xml_add(request, F_TYPE, T_CRM);
crm_xml_add(request, F_CRM_VERSION, CRM_FEATURE_SET);
crm_xml_add(request, F_CRM_MSG_TYPE, XML_ATTR_REQUEST);
crm_xml_add(request, F_CRM_REFERENCE, reference);
crm_xml_add(request, F_CRM_TASK, task);
crm_xml_add(request, F_CRM_SYS_TO, sys_to);
crm_xml_add(request, F_CRM_SYS_FROM, true_from);
/* HOSTTO will be ignored if it is to the DC anyway. */
if (host_to != NULL && strlen(host_to) > 0) {
crm_xml_add(request, F_CRM_HOST_TO, host_to);
}
if (msg_data != NULL) {
add_message_xml(request, F_CRM_DATA, msg_data);
}
free(reference);
free(true_from);
return request;
}
/*
* This method adds a copy of xml_response_data
*/
xmlNode *
create_reply_adv(xmlNode * original_request, xmlNode * xml_response_data, const char *origin)
{
xmlNode *reply = NULL;
const char *host_from = crm_element_value(original_request, F_CRM_HOST_FROM);
const char *sys_from = crm_element_value(original_request, F_CRM_SYS_FROM);
const char *sys_to = crm_element_value(original_request, F_CRM_SYS_TO);
const char *type = crm_element_value(original_request, F_CRM_MSG_TYPE);
const char *operation = crm_element_value(original_request, F_CRM_TASK);
const char *crm_msg_reference = crm_element_value(original_request, F_CRM_REFERENCE);
if (type == NULL) {
crm_err("Cannot create new_message, no message type in original message");
CRM_ASSERT(type != NULL);
return NULL;
#if 0
} else if (strcasecmp(XML_ATTR_REQUEST, type) != 0) {
crm_err("Cannot create new_message, original message was not a request");
return NULL;
#endif
}
reply = create_xml_node(NULL, __FUNCTION__);
if (reply == NULL) {
crm_err("Cannot create new_message, malloc failed");
return NULL;
}
crm_xml_add(reply, F_CRM_ORIGIN, origin);
crm_xml_add(reply, F_TYPE, T_CRM);
crm_xml_add(reply, F_CRM_VERSION, CRM_FEATURE_SET);
crm_xml_add(reply, F_CRM_MSG_TYPE, XML_ATTR_RESPONSE);
crm_xml_add(reply, F_CRM_REFERENCE, crm_msg_reference);
crm_xml_add(reply, F_CRM_TASK, operation);
/* since this is a reply, we reverse the from and to */
crm_xml_add(reply, F_CRM_SYS_TO, sys_from);
crm_xml_add(reply, F_CRM_SYS_FROM, sys_to);
/* HOSTTO will be ignored if it is to the DC anyway. */
if (host_from != NULL && strlen(host_from) > 0) {
crm_xml_add(reply, F_CRM_HOST_TO, host_from);
}
if (xml_response_data != NULL) {
add_message_xml(reply, F_CRM_DATA, xml_response_data);
}
return reply;
}
/* Libqb based IPC */
/* Server... */
GHashTable *client_connections = NULL;
crm_client_t *
crm_client_get(qb_ipcs_connection_t * c)
{
if (client_connections) {
return g_hash_table_lookup(client_connections, c);
}
crm_trace("No client found for %p", c);
return NULL;
}
crm_client_t *
crm_client_get_by_id(const char *id)
{
gpointer key;
crm_client_t *client;
GHashTableIter iter;
if (client_connections && id) {
g_hash_table_iter_init(&iter, client_connections);
while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) {
if (strcmp(client->id, id) == 0) {
return client;
}
}
}
crm_trace("No client found with id=%s", id);
return NULL;
}
const char *
crm_client_name(crm_client_t * c)
{
if (c == NULL) {
return "null";
} else if (c->name == NULL && c->id == NULL) {
return "unknown";
} else if (c->name == NULL) {
return c->id;
} else {
return c->name;
}
}
void
crm_client_init(void)
{
if (client_connections == NULL) {
crm_trace("Creating client hash table");
client_connections = g_hash_table_new(g_direct_hash, g_direct_equal);
}
}
void
crm_client_cleanup(void)
{
if (client_connections != NULL) {
int active = g_hash_table_size(client_connections);
if (active) {
crm_err("Exiting with %d active connections", active);
}
g_hash_table_destroy(client_connections); client_connections = NULL;
}
}
void
crm_client_disconnect_all(qb_ipcs_service_t *service)
{
qb_ipcs_connection_t *c = qb_ipcs_connection_first_get(service);
while (c != NULL) {
qb_ipcs_connection_t *last = c;
c = qb_ipcs_connection_next_get(service, last);
/* There really shouldn't be anyone connected at this point */
crm_notice("Disconnecting client %p, pid=%d...", last, crm_ipcs_client_pid(last));
qb_ipcs_disconnect(last);
qb_ipcs_connection_unref(last);
}
}
crm_client_t *
crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
{
static uid_t uid_server = 0;
static gid_t gid_cluster = 0;
crm_client_t *client = NULL;
CRM_LOG_ASSERT(c);
if (c == NULL) {
return NULL;
}
if (gid_cluster == 0) {
uid_server = getuid();
if(crm_user_lookup(CRM_DAEMON_USER, NULL, &gid_cluster) < 0) {
static bool have_error = FALSE;
if(have_error == FALSE) {
crm_warn("Could not find group for user %s", CRM_DAEMON_USER);
have_error = TRUE;
}
}
}
if(gid_cluster != 0 && gid_client != 0) {
uid_t best_uid = -1; /* Passing -1 to chown(2) means don't change */
if(uid_client == 0 || uid_server == 0) { /* Someone is priveliged, but the other may not be */
best_uid = QB_MAX(uid_client, uid_server);
crm_trace("Allowing user %u to clean up after disconnect", best_uid);
}
crm_trace("Giving access to group %u", gid_cluster);
qb_ipcs_connection_auth_set(c, best_uid, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
}
crm_client_init();
/* TODO: Do our own auth checking, return NULL if unauthorized */
client = calloc(1, sizeof(crm_client_t));
client->ipcs = c;
client->kind = CRM_CLIENT_IPC;
client->pid = crm_ipcs_client_pid(c);
client->id = crm_generate_uuid();
- crm_info("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id);
+ crm_debug("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id);
#if ENABLE_ACL
client->user = uid2username(uid_client);
#endif
g_hash_table_insert(client_connections, c, client);
return client;
}
void
crm_client_destroy(crm_client_t * c)
{
if (c == NULL) {
return;
}
if (client_connections) {
if (c->ipcs) {
crm_trace("Destroying %p/%p (%d remaining)",
c, c->ipcs, crm_hash_table_size(client_connections) - 1);
g_hash_table_remove(client_connections, c->ipcs);
} else {
crm_trace("Destroying remote connection %p (%d remaining)",
c, crm_hash_table_size(client_connections) - 1);
g_hash_table_remove(client_connections, c->id);
}
}
if (c->event_timer) {
g_source_remove(c->event_timer);
}
- crm_info("Destroying %d events", g_list_length(c->event_queue));
+ crm_debug("Destroying %d events", g_list_length(c->event_queue));
while (c->event_queue) {
struct iovec *event = c->event_queue->data;
c->event_queue = g_list_remove(c->event_queue, event);
free(event[0].iov_base);
free(event[1].iov_base);
free(event);
}
free(c->id);
free(c->name);
free(c->user);
if (c->remote) {
if (c->remote->auth_timeout) {
g_source_remove(c->remote->auth_timeout);
}
free(c->remote->buffer);
free(c->remote);
}
free(c);
}
int
crm_ipcs_client_pid(qb_ipcs_connection_t * c)
{
struct qb_ipcs_connection_stats stats;
stats.client_pid = 0;
qb_ipcs_connection_stats_get(c, &stats, 0);
return stats.client_pid;
}
xmlNode *
crm_ipcs_recv(crm_client_t * c, void *data, size_t size, uint32_t * id, uint32_t * flags)
{
xmlNode *xml = NULL;
char *uncompressed = NULL;
char *text = ((char *)data) + sizeof(struct crm_ipc_response_header);
struct crm_ipc_response_header *header = data;
if (id) {
*id = ((struct qb_ipc_response_header *)data)->id;
}
if (flags) {
*flags = header->flags;
}
if(header->version > PCMK_IPC_VERSION) {
crm_err("Filtering incompatible v%d IPC message, we only support versions <= %d",
header->version, PCMK_IPC_VERSION);
return NULL;
}
if (header->size_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->size_uncompressed;
uncompressed = calloc(1, size_u);
crm_trace("Decompressing message data %d bytes into %d bytes",
header->size_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0);
text = uncompressed;
if (rc != BZ_OK) {
crm_err("Decompression failed: %s (%d)", bz2_strerror(rc), rc);
free(uncompressed);
return NULL;
}
}
CRM_ASSERT(text[header->size_uncompressed - 1] == 0);
crm_trace("Received %.200s", text);
xml = string2xml(text);
free(uncompressed);
return xml;
}
ssize_t crm_ipcs_flush_events(crm_client_t * c);
static gboolean
crm_ipcs_flush_events_cb(gpointer data)
{
crm_client_t *c = data;
c->event_timer = 0;
crm_ipcs_flush_events(c);
return FALSE;
}
ssize_t
crm_ipcs_flush_events(crm_client_t * c)
{
int sent = 0;
ssize_t rc = 0;
int queue_len = 0;
if (c == NULL) {
return pcmk_ok;
} else if (c->event_timer) {
/* There is already a timer, wait until it goes off */
crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer);
return pcmk_ok;
}
queue_len = g_list_length(c->event_queue);
while (c->event_queue && sent < 100) {
struct crm_ipc_response_header *header = NULL;
struct iovec *event = c->event_queue->data;
rc = qb_ipcs_event_sendv(c->ipcs, event, 2);
if (rc < 0) {
break;
}
sent++;
header = event[0].iov_base;
if (header->size_compressed) {
crm_trace("Event %d to %p[%d] (%d compressed bytes) sent",
header->qb.id, c->ipcs, c->pid, rc);
} else {
crm_trace("Event %d to %p[%d] (%d bytes) sent: %.120s",
header->qb.id, c->ipcs, c->pid, rc, event[1].iov_base);
}
c->event_queue = g_list_remove(c->event_queue, event);
free(event[0].iov_base);
free(event[1].iov_base);
free(event);
}
queue_len -= sent;
if (sent > 0 || c->event_queue) {
crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%d)",
sent, queue_len, c->ipcs, c->pid, pcmk_strerror(rc < 0 ? rc : 0), rc);
}
if (c->event_queue) {
if (queue_len % 100 == 0 && queue_len > 99) {
crm_warn("Event queue for %p[%d] has grown to %d", c->ipcs, c->pid, queue_len);
} else if (queue_len > 500) {
crm_err("Evicting slow client %p[%d]: event queue reached %d entries",
c->ipcs, c->pid, queue_len);
qb_ipcs_disconnect(c->ipcs);
return rc;
}
c->event_timer = g_timeout_add(1000 + 100 * queue_len, crm_ipcs_flush_events_cb, c);
}
return rc;
}
ssize_t
crm_ipc_prepare(uint32_t request, xmlNode * message, struct iovec ** result, int32_t max_send_size)
{
static int biggest = 0;
struct iovec *iov;
unsigned int total = 0;
char *compressed = NULL;
char *buffer = dump_xml_unformatted(message);
struct crm_ipc_response_header *header = calloc(1, sizeof(struct crm_ipc_response_header));
CRM_ASSERT(result != NULL);
crm_ipc_init();
if (max_send_size == 0) {
max_send_size = ipc_buffer_max;
}
CRM_LOG_ASSERT(max_send_size != 0);
*result = NULL;
iov = calloc(2, sizeof(struct iovec));
iov[0].iov_len = hdr_offset;
iov[0].iov_base = header;
header->version = PCMK_IPC_VERSION;
header->size_uncompressed = 1 + strlen(buffer);
total = iov[0].iov_len + header->size_uncompressed;
if (total < max_send_size) {
iov[1].iov_base = buffer;
iov[1].iov_len = header->size_uncompressed;
} else {
unsigned int new_size = 0;
if (crm_compress_string
(buffer, header->size_uncompressed, max_send_size, &compressed, &new_size)) {
header->flags |= crm_ipc_compressed;
header->size_compressed = new_size;
iov[1].iov_len = header->size_compressed;
iov[1].iov_base = compressed;
free(buffer);
if (header->size_compressed > biggest) {
biggest = 2 * QB_MAX(header->size_compressed, biggest);
}
} else {
ssize_t rc = -EMSGSIZE;
crm_log_xml_trace(message, "EMSGSIZE");
biggest = 2 * QB_MAX(header->size_uncompressed, biggest);
crm_err
("Could not compress the message into less than the configured ipc limit (%d bytes)."
"Set PCMK_ipc_buffer to a higher value (%d bytes suggested)", max_send_size,
biggest);
free(compressed);
free(buffer);
free(header);
free(iov);
return rc;
}
}
header->qb.size = iov[0].iov_len + iov[1].iov_len;
header->qb.id = (int32_t)request; /* Replying to a specific request */
*result = iov;
return header->qb.size;
}
ssize_t
crm_ipcs_sendv(crm_client_t * c, struct iovec * iov, enum crm_ipc_flags flags)
{
ssize_t rc;
static uint32_t id = 1;
struct crm_ipc_response_header *header = iov[0].iov_base;
if (flags & crm_ipc_proxied) {
/* _ALL_ replies to proxied connections need to be sent as events */
flags |= crm_ipc_server_event;
}
if (flags & crm_ipc_server_event) {
header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */
if (flags & crm_ipc_server_free) {
crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid);
c->event_queue = g_list_append(c->event_queue, iov);
} else {
struct iovec *iov_copy = calloc(2, sizeof(struct iovec));
crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid);
iov_copy[0].iov_len = iov[0].iov_len;
iov_copy[0].iov_base = malloc(iov[0].iov_len);
memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len);
iov_copy[1].iov_len = iov[1].iov_len;
iov_copy[1].iov_base = malloc(iov[1].iov_len);
memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len);
c->event_queue = g_list_append(c->event_queue, iov_copy);
}
} else {
CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */
rc = qb_ipcs_response_sendv(c->ipcs, iov, 2);
if (rc < header->qb.size) {
crm_notice("Response %d to %p[%d] (%d bytes) failed: %s (%d)",
header->qb.id, c->ipcs, c->pid, header->qb.size, pcmk_strerror(rc), rc);
} else {
crm_trace("Response %d sent, %d bytes to %p[%d]", header->qb.id, rc, c->ipcs, c->pid);
}
if (flags & crm_ipc_server_free) {
free(iov[0].iov_base);
free(iov[1].iov_base);
free(iov);
}
}
if (flags & crm_ipc_server_event) {
rc = crm_ipcs_flush_events(c);
} else {
crm_ipcs_flush_events(c);
}
if (rc == -EPIPE || rc == -ENOTCONN) {
crm_trace("Client %p disconnected", c->ipcs);
}
return rc;
}
ssize_t
crm_ipcs_send(crm_client_t * c, uint32_t request, xmlNode * message,
enum crm_ipc_flags flags)
{
struct iovec *iov = NULL;
ssize_t rc = 0;
if(c == NULL) {
return -EDESTADDRREQ;
}
crm_ipc_init();
rc = crm_ipc_prepare(request, message, &iov, ipc_buffer_max);
if (rc > 0) {
rc = crm_ipcs_sendv(c, iov, flags | crm_ipc_server_free);
} else {
free(iov);
crm_notice("Message to %p[%d] failed: %s (%d)",
c->ipcs, c->pid, pcmk_strerror(rc), rc);
}
return rc;
}
void
crm_ipcs_send_ack(crm_client_t * c, uint32_t request, uint32_t flags, const char *tag, const char *function,
int line)
{
if (flags & crm_ipc_client_response) {
xmlNode *ack = create_xml_node(NULL, tag);
crm_trace("Ack'ing msg from %s (%p)", crm_client_name(c), c);
c->request_id = 0;
crm_xml_add(ack, "function", function);
crm_xml_add_int(ack, "line", line);
crm_ipcs_send(c, request, ack, flags);
free_xml(ack);
}
}
/* Client... */
#define MIN_MSG_SIZE 12336 /* sizeof(struct qb_ipc_connection_response) */
#define MAX_MSG_SIZE 128*1024 /* 128k default */
struct crm_ipc_s {
struct pollfd pfd;
/* the max size we can send/receive over ipc */
int max_buf_size;
/* Size of the allocated 'buffer' */
int buf_size;
int msg_size;
int need_reply;
char *buffer;
char *name;
qb_ipcc_connection_t *ipc;
};
static unsigned int
pick_ipc_buffer(int max)
{
static int global_max = 0;
if(global_max == 0) {
const char *env = getenv("PCMK_ipc_buffer");
if (env) {
global_max = crm_parse_int(env, "0");
} else {
global_max = MAX_MSG_SIZE;
}
}
return QB_MAX(max, global_max);
}
crm_ipc_t *
crm_ipc_new(const char *name, size_t max_size)
{
crm_ipc_t *client = NULL;
client = calloc(1, sizeof(crm_ipc_t));
client->name = strdup(name);
client->buf_size = pick_ipc_buffer(max_size);
client->buffer = malloc(client->buf_size);
/* Clients initiating connection pick the max buf size */
client->max_buf_size = client->buf_size;
client->pfd.fd = -1;
client->pfd.events = POLLIN;
client->pfd.revents = 0;
return client;
}
bool
crm_ipc_connect(crm_ipc_t * client)
{
client->need_reply = FALSE;
client->ipc = qb_ipcc_connect(client->name, client->buf_size);
if (client->ipc == NULL) {
crm_perror(LOG_INFO, "Could not establish %s connection", client->name);
return FALSE;
}
client->pfd.fd = crm_ipc_get_fd(client);
if (client->pfd.fd < 0) {
crm_perror(LOG_INFO, "Could not obtain file descriptor for %s connection", client->name);
return FALSE;
}
qb_ipcc_context_set(client->ipc, client);
#ifdef HAVE_IPCS_GET_BUFFER_SIZE
client->max_buf_size = qb_ipcc_get_buffer_size(client->ipc);
if (client->max_buf_size < client->buf_size) {
free(client->buffer);
client->buffer = calloc(1, client->max_buf_size);
client->buf_size = client->max_buf_size;
}
#endif
return TRUE;
}
void
crm_ipc_close(crm_ipc_t * client)
{
if (client) {
crm_trace("Disconnecting %s IPC connection %p (%p.%p)", client->name, client, client->ipc);
if (client->ipc) {
qb_ipcc_connection_t *ipc = client->ipc;
client->ipc = NULL;
qb_ipcc_disconnect(ipc);
}
}
}
void
crm_ipc_destroy(crm_ipc_t * client)
{
if (client) {
if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
crm_notice("Destroying an active IPC connection to %s", client->name);
/* The next line is basically unsafe
*
* If this connection was attached to mainloop and mainloop is active,
* the 'disconnected' callback will end up back here and we'll end
* up free'ing the memory twice - something that can still happen
* even without this if we destroy a connection and it closes before
* we call exit
*/
/* crm_ipc_close(client); */
}
crm_trace("Destroying IPC connection to %s: %p", client->name, client);
free(client->buffer);
free(client->name);
free(client);
}
}
int
crm_ipc_get_fd(crm_ipc_t * client)
{
int fd = 0;
CRM_ASSERT(client != NULL);
if (client->ipc && qb_ipcc_fd_get(client->ipc, &fd) == 0) {
return fd;
}
crm_perror(LOG_ERR, "Could not obtain file IPC descriptor for %s", client->name);
return -EINVAL;
}
bool
crm_ipc_connected(crm_ipc_t * client)
{
bool rc = FALSE;
if (client == NULL) {
crm_trace("No client");
return FALSE;
} else if (client->ipc == NULL) {
crm_trace("No connection");
return FALSE;
} else if (client->pfd.fd < 0) {
crm_trace("Bad descriptor");
return FALSE;
}
rc = qb_ipcc_is_connected(client->ipc);
if (rc == FALSE) {
client->pfd.fd = -EINVAL;
}
return rc;
}
int
crm_ipc_ready(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
if (crm_ipc_connected(client) == FALSE) {
return -ENOTCONN;
}
client->pfd.revents = 0;
return poll(&(client->pfd), 1, 0);
}
static int
crm_ipc_decompress(crm_ipc_t * client)
{
struct crm_ipc_response_header *header = (struct crm_ipc_response_header *)(void*)client->buffer;
if (header->size_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->size_uncompressed;
/* never let buf size fall below our max size required for ipc reads. */
unsigned int new_buf_size = QB_MAX((hdr_offset + size_u), client->max_buf_size);
char *uncompressed = calloc(1, new_buf_size);
crm_trace("Decompressing message data %d bytes into %d bytes",
header->size_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed + hdr_offset, &size_u,
client->buffer + hdr_offset, header->size_compressed, 1, 0);
if (rc != BZ_OK) {
crm_err("Decompression failed: %s (%d)", bz2_strerror(rc), rc);
free(uncompressed);
return -EILSEQ;
}
/*
* This assert no longer holds true. For an identical msg, some clients may
* require compression, and others may not. If that same msg (event) is sent
* to multiple clients, it could result in some clients receiving a compressed
* msg even though compression was not explicitly required for them.
*
* CRM_ASSERT((header->size_uncompressed + hdr_offset) >= ipc_buffer_max);
*/
CRM_ASSERT(size_u == header->size_uncompressed);
memcpy(uncompressed, client->buffer, hdr_offset); /* Preserve the header */
header = (struct crm_ipc_response_header *)(void*)uncompressed;
free(client->buffer);
client->buf_size = new_buf_size;
client->buffer = uncompressed;
}
CRM_ASSERT(client->buffer[hdr_offset + header->size_uncompressed - 1] == 0);
return pcmk_ok;
}
long
crm_ipc_read(crm_ipc_t * client)
{
struct crm_ipc_response_header *header = NULL;
CRM_ASSERT(client != NULL);
CRM_ASSERT(client->ipc != NULL);
CRM_ASSERT(client->buffer != NULL);
crm_ipc_init();
client->buffer[0] = 0;
client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer, client->buf_size - 1, 0);
if (client->msg_size >= 0) {
int rc = crm_ipc_decompress(client);
if (rc != pcmk_ok) {
return rc;
}
header = (struct crm_ipc_response_header *)(void*)client->buffer;
if(header->version > PCMK_IPC_VERSION) {
crm_err("Filtering incompatible v%d IPC message, we only support versions <= %d",
header->version, PCMK_IPC_VERSION);
return -EBADMSG;
}
crm_trace("Received %s event %d, size=%d, rc=%d, text: %.100s",
client->name, header->qb.id, header->qb.size, client->msg_size,
client->buffer + hdr_offset);
} else {
crm_trace("No message from %s received: %s", client->name, pcmk_strerror(client->msg_size));
}
if (crm_ipc_connected(client) == FALSE || client->msg_size == -ENOTCONN) {
crm_err("Connection to %s failed", client->name);
}
if (header) {
/* Data excluding the header */
return header->size_uncompressed;
}
return -ENOMSG;
}
const char *
crm_ipc_buffer(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
return client->buffer + sizeof(struct crm_ipc_response_header);
}
const char *
crm_ipc_name(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
return client->name;
}
static int
internal_ipc_send_recv(crm_ipc_t * client, const void *iov)
{
int rc = 0;
do {
rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer, client->buf_size, -1);
} while (rc == -EAGAIN && crm_ipc_connected(client));
return rc;
}
static int
internal_ipc_send_request(crm_ipc_t * client, const void *iov, int ms_timeout)
{
int rc = 0;
time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
do {
rc = qb_ipcc_sendv(client->ipc, iov, 2);
} while (rc == -EAGAIN && time(NULL) < timeout && crm_ipc_connected(client));
return rc;
}
static int
internal_ipc_get_reply(crm_ipc_t * client, int request_id, int ms_timeout)
{
time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
int rc = 0;
crm_ipc_init();
/* get the reply */
crm_trace("client %s waiting on reply to msg id %d", client->name, request_id);
do {
rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
if (rc > 0) {
struct crm_ipc_response_header *hdr = NULL;
int rc = crm_ipc_decompress(client);
if (rc != pcmk_ok) {
return rc;
}
hdr = (struct crm_ipc_response_header *)(void*)client->buffer;
if (hdr->qb.id == request_id) {
/* Got it */
break;
} else if (hdr->qb.id < request_id) {
xmlNode *bad = string2xml(crm_ipc_buffer(client));
crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id);
crm_log_xml_notice(bad, "OldIpcReply");
} else {
xmlNode *bad = string2xml(crm_ipc_buffer(client));
crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
crm_log_xml_notice(bad, "ImpossibleReply");
CRM_ASSERT(hdr->qb.id <= request_id);
}
} else if (crm_ipc_connected(client) == FALSE) {
crm_err("Server disconnected client %s while waiting for msg id %d", client->name,
request_id);
break;
}
} while (time(NULL) < timeout);
return rc;
}
int
crm_ipc_send(crm_ipc_t * client, xmlNode * message, enum crm_ipc_flags flags, int32_t ms_timeout,
xmlNode ** reply)
{
long rc = 0;
struct iovec *iov;
static uint32_t id = 0;
static int factor = 8;
struct crm_ipc_response_header *header;
crm_ipc_init();
if (client == NULL) {
crm_notice("Invalid connection");
return -ENOTCONN;
} else if (crm_ipc_connected(client) == FALSE) {
/* Don't even bother */
crm_notice("Connection to %s closed", client->name);
return -ENOTCONN;
}
if (ms_timeout == 0) {
ms_timeout = 5000;
}
if (client->need_reply) {
crm_trace("Trying again to obtain pending reply from %s", client->name);
rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
if (rc < 0) {
crm_warn("Sending to %s (%p) is disabled until pending reply is received", client->name,
client->ipc);
return -EALREADY;
} else {
crm_notice("Lost reply from %s (%p) finally arrived, sending re-enabled", client->name,
client->ipc);
client->need_reply = FALSE;
}
}
id++;
CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */
rc = crm_ipc_prepare(id, message, &iov, ipc_buffer_max);
if(rc < 0) {
return rc;
}
header = iov[0].iov_base;
header->flags |= flags;
if(is_set(flags, crm_ipc_proxied)) {
/* Don't look for a synchronous response */
clear_bit(flags, crm_ipc_client_response);
}
if(header->size_compressed) {
if(factor < 10 && (ipc_buffer_max / 10) < (rc / factor)) {
crm_notice("Compressed message exceeds %d0%% of the configured ipc limit (%d bytes), "
"consider setting PCMK_ipc_buffer to %d or higher",
factor, ipc_buffer_max, 2*ipc_buffer_max);
factor++;
}
}
crm_trace("Sending from client: %s request id: %d bytes: %u timeout:%d msg...",
client->name, header->qb.id, header->qb.size, ms_timeout);
if (ms_timeout > 0 || is_not_set(flags, crm_ipc_client_response)) {
rc = internal_ipc_send_request(client, iov, ms_timeout);
if (rc <= 0) {
crm_trace("Failed to send from client %s request %d with %u bytes...",
client->name, header->qb.id, header->qb.size);
goto send_cleanup;
} else if (is_not_set(flags, crm_ipc_client_response)) {
crm_trace("Message sent, not waiting for reply to %d from %s to %u bytes...",
header->qb.id, client->name, header->qb.size);
goto send_cleanup;
}
rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout);
if (rc < 0) {
/* No reply, for now, disable sending
*
* The alternative is to close the connection since we don't know
* how to detect and discard out-of-sequence replies
*
* TODO - implement the above
*/
client->need_reply = TRUE;
}
} else {
rc = internal_ipc_send_recv(client, iov);
}
if (rc > 0) {
struct crm_ipc_response_header *hdr = (struct crm_ipc_response_header *)(void*)client->buffer;
crm_trace("Received response %d, size=%d, rc=%ld, text: %.200s", hdr->qb.id, hdr->qb.size,
rc, crm_ipc_buffer(client));
if (reply) {
*reply = string2xml(crm_ipc_buffer(client));
}
} else {
crm_trace("Response not received: rc=%ld, errno=%d", rc, errno);
}
send_cleanup:
if (crm_ipc_connected(client) == FALSE) {
crm_notice("Connection to %s closed: %s (%ld)", client->name, pcmk_strerror(rc), rc);
} else if (rc == -ETIMEDOUT) {
crm_warn("Request %d to %s (%p) failed: %s (%ld) after %dms",
header->qb.id, client->name, client->ipc, pcmk_strerror(rc), rc, ms_timeout);
crm_write_blackbox(0, NULL);
} else if (rc <= 0) {
crm_warn("Request %d to %s (%p) failed: %s (%ld)",
header->qb.id, client->name, client->ipc, pcmk_strerror(rc), rc);
}
free(header);
free(iov[1].iov_base);
free(iov);
return rc;
}
/* Utils */
xmlNode *
create_hello_message(const char *uuid,
const char *client_name, const char *major_version, const char *minor_version)
{
xmlNode *hello_node = NULL;
xmlNode *hello = NULL;
if (uuid == NULL || strlen(uuid) == 0
|| client_name == NULL || strlen(client_name) == 0
|| major_version == NULL || strlen(major_version) == 0
|| minor_version == NULL || strlen(minor_version) == 0) {
crm_err("Missing fields, Hello message will not be valid.");
return NULL;
}
hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
crm_xml_add(hello_node, "major_version", major_version);
crm_xml_add(hello_node, "minor_version", minor_version);
crm_xml_add(hello_node, "client_name", client_name);
crm_xml_add(hello_node, "client_uuid", uuid);
crm_trace("creating hello message");
hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
free_xml(hello_node);
return hello;
}