diff --git a/attrd/commands.c b/attrd/commands.c
index 9c87a55e63..287bd4353b 100644
--- a/attrd/commands.c
+++ b/attrd/commands.c
@@ -1,728 +1,768 @@
 /*
  * Copyright (C) 2013 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <crm_internal.h>
 
 #include <glib.h>
 
 #include <crm/msg_xml.h>
 #include <crm/cluster.h>
 #include <crm/cib.h>
 #include <crm/cluster/internal.h>
 #include <crm/cluster/election.h>
 #include <crm/cib/internal.h>
 
 #include <internal.h>
 
+#define ATTRD_PROTOCOL_VERSION "1"
+
 int last_cib_op_done = 0;
 char *peer_writer = NULL;
 GHashTable *attributes = NULL;
 
 typedef struct attribute_s {
     char *uuid; /* TODO: Remove if at all possible */
     char *id;
     char *set;
 
     GHashTable *values;
 
     int update;
     int timeout_ms;
     bool changed;
     bool unknown_peer_uuids;
     mainloop_timer_t *timer;
 
     char *user;
 
 } attribute_t;
 
 typedef struct attribute_value_s {
         uint32_t nodeid;
         gboolean is_remote;
         char *nodename;
         char *current;
         char *requested;
         char *stored;
 } attribute_value_t;
 
 
 void write_attribute(attribute_t *a);
 void write_or_elect_attribute(attribute_t *a);
 void attrd_peer_update(crm_node_t *peer, xmlNode *xml, bool filter);
 void attrd_peer_sync(crm_node_t *peer, xmlNode *xml);
 void attrd_peer_remove(const char *host, const char *source);
 
+static gboolean
+send_attrd_message(crm_node_t * node, xmlNode * data)
+{
+    crm_xml_add(data, F_TYPE, T_ATTRD);
+    crm_xml_add(data, F_ATTRD_IGNORE_LOCALLY, "atomic-version"); /* Tell older versions to ignore our messages */
+    crm_xml_add(data, F_ATTRD_VERSION, ATTRD_PROTOCOL_VERSION);
+    crm_xml_add_int(data, F_ATTRD_WRITER, election_state(writer));
+
+    return send_cluster_message(node, crm_msg_attrd, data, TRUE);
+}
+
 static gboolean
 attribute_timer_cb(gpointer data)
 {
     attribute_t *a = data;
     crm_trace("Dampen interval expired for %s in state %d", a->id, election_state(writer));
     write_or_elect_attribute(a);
     return FALSE;
 }
 
 static void
 free_attribute_value(gpointer data)
 {
     attribute_value_t *v = data;
 
     free(v->nodename);
     free(v->current);
     free(v->requested);
     free(v->stored);
     free(v);
 }
 
 void
 free_attribute(gpointer data)
 {
     attribute_t *a = data;
     if(a) {
         free(a->id);
         free(a->set);
         free(a->uuid);
         free(a->user);
 
         mainloop_timer_del(a->timer);
         g_hash_table_destroy(a->values);
 
         free(a);
     }
 }
 
 xmlNode *
 build_attribute_xml(
     xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int timeout_ms, const char *user,
     const char *peer, uint32_t peerid, const char *value)
 {
     xmlNode *xml = create_xml_node(parent, __FUNCTION__);
 
     crm_xml_add(xml, F_ATTRD_ATTRIBUTE, name);
     crm_xml_add(xml, F_ATTRD_SET, set);
     crm_xml_add(xml, F_ATTRD_KEY, uuid);
     crm_xml_add(xml, F_ATTRD_USER, user);
     crm_xml_add(xml, F_ATTRD_HOST, peer);
     crm_xml_add_int(xml, F_ATTRD_HOST_ID, peerid);
     crm_xml_add(xml, F_ATTRD_VALUE, value);
     crm_xml_add_int(xml, F_ATTRD_DAMPEN, timeout_ms/1000);
 
     return xml;
 }
 
 static attribute_t *
 create_attribute(xmlNode *xml)
 {
     int dampen = 0;
     const char *value = crm_element_value_copy(xml, F_ATTRD_DAMPEN);
     attribute_t *a = calloc(1, sizeof(attribute_t));
 
     a->id      = crm_element_value_copy(xml, F_ATTRD_ATTRIBUTE);
     a->set     = crm_element_value_copy(xml, F_ATTRD_SET);
     a->uuid    = crm_element_value_copy(xml, F_ATTRD_KEY);
     a->values = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute_value);
 
 #if ENABLE_ACL
     crm_trace("Performing all %s operations as user '%s'", a->id, a->user);
     a->user = crm_element_value_copy(xml, F_ATTRD_USER);
 #endif
 
     if(value) {
         dampen = crm_get_msec(value);
         crm_trace("Created attribute %s with delay %dms (%s)", a->id, dampen, value);
     } else {
         crm_trace("Created attribute %s with no delay", a->id);
     }
 
     if(dampen > 0) {
         a->timeout_ms = dampen;
         a->timer = mainloop_timer_add(strdup(a->id), a->timeout_ms, FALSE, attribute_timer_cb, a);
     }
 
     g_hash_table_replace(attributes, a->id, a);
     return a;
 }
 
 void
 attrd_client_message(crm_client_t *client, xmlNode *xml)
 {
     bool broadcast = FALSE;
     static int plus_plus_len = 5;
     const char *op = crm_element_value(xml, F_ATTRD_TASK);
 
     if(safe_str_eq(op, "peer-remove")) {
         const char *host = crm_element_value(xml, F_ATTRD_HOST);
 
         crm_info("Client %s is requesting all values for %s be removed", client->name, host);
         if(host) {
             broadcast = TRUE;
         }
 
     } else if(safe_str_eq(op, "update")) {
         attribute_t *a = NULL;
         attribute_value_t *v = NULL;
         char *key = crm_element_value_copy(xml, F_ATTRD_KEY);
         char *set = crm_element_value_copy(xml, F_ATTRD_SET);
         char *host = crm_element_value_copy(xml, F_ATTRD_HOST);
         const char *attr = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
         const char *value = crm_element_value(xml, F_ATTRD_VALUE);
 
         a = g_hash_table_lookup(attributes, attr);
 
         if(host == NULL) {
             crm_trace("Inferring host");
             host = strdup(attrd_cluster->uname);
             crm_xml_add(xml, F_ATTRD_HOST, host);
             crm_xml_add_int(xml, F_ATTRD_HOST_ID, attrd_cluster->nodeid);
         }
 
         if (value) {
             int offset = 1;
             int int_value = 0;
             int value_len = strlen(value);
 
             if (value_len < (plus_plus_len + 2)
                 || value[plus_plus_len] != '+'
                 || (value[plus_plus_len + 1] != '+' && value[plus_plus_len + 1] != '=')) {
                 goto send;
             }
 
             if(a) {
                 v = g_hash_table_lookup(a->values, host);
             }
             if(v) {
                 int_value = char2score(v->current);
             }
 
             if (value[plus_plus_len + 1] != '+') {
                 const char *offset_s = value + (plus_plus_len + 2);
 
                 offset = char2score(offset_s);
             }
             int_value += offset;
 
             if (int_value > INFINITY) {
                 int_value = INFINITY;
             }
 
             crm_info("Expanded %s=%s to %d", attr, value, int_value);
             crm_xml_add_int(xml, F_ATTRD_VALUE, int_value);
         }
 
       send:
 
         if(peer_writer == NULL && election_state(writer) != election_in_progress) {
             crm_info("Starting an election to determine the writer");
             election_vote(writer);
         }
 
         crm_info("Broadcasting %s[%s] = %s%s", attr, host, value, election_state(writer) == election_won?" (writer)":"");
         broadcast = TRUE;
 
         free(key);
         free(set);
         free(host);
     }
 
     if(broadcast) {
-        crm_xml_add_int(xml, F_ATTRD_WRITER, election_state(writer));
-        send_cluster_message(NULL, crm_msg_attrd, xml, TRUE);
+        send_attrd_message(NULL, xml);
     }
 }
 
 void
 attrd_peer_message(crm_node_t *peer, xmlNode *xml)
 {
     int peer_state = 0;
+    const char *v = crm_element_value(xml, F_ATTRD_VERSION);
     const char *op = crm_element_value(xml, F_ATTRD_TASK);
     const char *election_op = crm_element_value(xml, F_CRM_TASK);
 
     if(election_op) {
         enum election_result rc = 0;
 
         crm_xml_add(xml, F_CRM_HOST_FROM, peer->uname);
         rc = election_count_vote(writer, xml, TRUE);
         switch(rc) {
             case election_start:
                 free(peer_writer);
                 peer_writer = NULL;
                 election_vote(writer);
                 break;
             case election_lost:
                 free(peer_writer);
                 peer_writer = strdup(peer->uname);
                 break;
             default:
                 election_check(writer);
                 break;
         }
         return;
+
+    } else if(v == NULL) {
+        /* From the non-atomic version */
+        if(safe_str_eq(op, "update")) {
+            const char *name = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
+
+            crm_trace("Compatibility update of %s from %s", name, peer->uname);
+            attrd_peer_update(peer, xml, FALSE);
+
+        } else if(safe_str_eq(op, "flush")) {
+            const char *name = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
+            attribute_t *a = g_hash_table_lookup(attributes, name);
+
+            if(a) {
+                crm_trace("Compatibility write-out of %s for %s from %s", a->id, op, peer->uname);
+                write_or_elect_attribute(a);
+            }
+
+        } else if(safe_str_eq(op, "refresh")) {
+            GHashTableIter aIter;
+            attribute_t *a = NULL;
+
+            g_hash_table_iter_init(&aIter, attributes);
+            while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
+                crm_trace("Compatibility write-out of %s for %s from %s", a->id, op, peer->uname);
+                write_or_elect_attribute(a);
+            }
+        }
     }
 
     crm_element_value_int(xml, F_ATTRD_WRITER, &peer_state);
     if(election_state(writer) == election_won
        && peer_state == election_won
        && safe_str_neq(peer->uname, attrd_cluster->uname)) {
         crm_notice("Detected another attribute writer: %s", peer->uname);
         election_vote(writer);
 
     } else if(peer_state == election_won) {
         if(peer_writer == NULL) {
             peer_writer = strdup(peer->uname);
             crm_notice("Recorded attribute writer: %s", peer->uname);
 
         } else if(safe_str_neq(peer->uname, peer_writer)) {
             crm_notice("Recorded new attribute writer: %s (was %s)", peer->uname, peer_writer);
             free(peer_writer);
             peer_writer = strdup(peer->uname);
         }
     }
 
     if(safe_str_eq(op, "update")) {
         attrd_peer_update(peer, xml, FALSE);
 
     } else if(safe_str_eq(op, "sync")) {
         attrd_peer_sync(peer, xml);
 
     } else if(safe_str_eq(op, "peer-remove")) {
         const char *host = crm_element_value(xml, F_ATTRD_HOST);
         attrd_peer_remove(host, peer->uname);
 
     } else if(safe_str_eq(op, "sync-response")
               && safe_str_neq(peer->uname, attrd_cluster->uname)) {
         xmlNode *child = NULL;
 
         crm_notice("Processing %s from %s", op, peer->uname);
         for (child = __xml_first_child(xml); child != NULL; child = __xml_next(child)) {
             attrd_peer_update(peer, child, TRUE);
         }
     }
 }
 
 void
 attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
 {
     GHashTableIter aIter;
     GHashTableIter vIter;
 
     attribute_t *a = NULL;
     attribute_value_t *v = NULL;
     xmlNode *sync = create_xml_node(NULL, __FUNCTION__);
 
     crm_xml_add(sync, F_ATTRD_TASK, "sync-response");
 
     g_hash_table_iter_init(&aIter, attributes);
     while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
         g_hash_table_iter_init(&vIter, a->values);
         while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
             crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename, v->current, peer?peer->uname:"everyone");
             build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, v->nodename, v->nodeid, v->current);
         }
     }
 
     crm_debug("Syncing values to %s", peer?peer->uname:"everyone");
-    crm_xml_add_int(sync, F_ATTRD_WRITER, election_state(writer));
-    send_cluster_message(peer, crm_msg_attrd, sync, TRUE);
+    send_attrd_message(peer, sync);
     free_xml(sync);
 }
 
 void
 attrd_peer_remove(const char *host, const char *source)
 {
     attribute_t *a = NULL;
     GHashTableIter aIter;
 
     crm_notice("Removing all %s attributes for %s", host, source);
     if(host == NULL) {
         return;
     }
 
     g_hash_table_iter_init(&aIter, attributes);
     while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
         if(g_hash_table_remove(a->values, host)) {
             crm_debug("Removed %s[%s] for %s", a->id, host, source);
         }
     }
 
     /* if this matches a remote peer, it will be removed from the cache */
     crm_remote_peer_cache_remove(host);
 }
 
 void
 attrd_peer_update(crm_node_t *peer, xmlNode *xml, bool filter)
 {
     bool changed = FALSE;
     attribute_value_t *v = NULL;
 
     const char *host = crm_element_value(xml, F_ATTRD_HOST);
     const char *attr = crm_element_value(xml, F_ATTRD_ATTRIBUTE);
     const char *value = crm_element_value(xml, F_ATTRD_VALUE);
 
     attribute_t *a = g_hash_table_lookup(attributes, attr);
 
     if(a == NULL) {
         a = create_attribute(xml);
     }
 
     v = g_hash_table_lookup(a->values, host);
 
     if(v == NULL) {
         crm_trace("Setting %s[%s] to %s from %s", attr, host, value, peer->uname);
         v = calloc(1, sizeof(attribute_value_t));
         if(value) {
             v->current = strdup(value);
         }
         v->nodename = strdup(host);
         crm_element_value_int(xml, F_ATTRD_IS_REMOTE, &v->is_remote);
         g_hash_table_replace(a->values, v->nodename, v);
 
         if (v->is_remote == TRUE) {
             crm_remote_peer_cache_add(host);
         }
 
         changed = TRUE;
 
     } else if(filter
               && safe_str_neq(v->current, value)
               && safe_str_eq(host, attrd_cluster->uname)) {
         xmlNode *sync = create_xml_node(NULL, __FUNCTION__);
         crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s",
                    a->id, host, v->current, value, peer->uname);
 
         crm_xml_add(sync, F_ATTRD_TASK, "sync-response");
         v = g_hash_table_lookup(a->values, host);
         build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms, a->user, v->nodename, v->nodeid, v->current);
 
         crm_xml_add_int(sync, F_ATTRD_WRITER, election_state(writer));
-        send_cluster_message(peer, crm_msg_attrd, sync, TRUE);
+        send_attrd_message(peer, sync);
         free_xml(sync);
 
     } else if(safe_str_neq(v->current, value)) {
         crm_info("Setting %s[%s]: %s -> %s from %s", attr, host, v->current, value, peer->uname);
         free(v->current);
         if(value) {
             v->current = strdup(value);
         } else {
             v->current = NULL;
         }
         changed = TRUE;
 
     } else {
         crm_trace("Unchanged %s[%s] from %s is %s", attr, host, peer->uname, value);
     }
 
     a->changed |= changed;
 
     /* this only involves cluster nodes. */
     if(v->nodeid == 0 && (v->is_remote == FALSE)) {
         if(crm_element_value_int(xml, F_ATTRD_HOST_ID, (int*)&v->nodeid) == 0) {
             /* Create the name/id association */
             crm_node_t *peer = crm_get_peer(v->nodeid, host);
             crm_trace("We know %s's node id now: %s", peer->uname, peer->uuid);
             if(election_state(writer) == election_won) {
                 write_attributes(FALSE, TRUE);
                 return;
             }
         }
     }
 
     if(changed) {
         if(a->timer) {
             crm_trace("Delayed write out (%dms) for %s", a->timeout_ms, a->id);
             mainloop_timer_start(a->timer);
         } else {
             write_or_elect_attribute(a);
         }
     }
 }
 
 void
 write_or_elect_attribute(attribute_t *a)
 {
     enum election_result rc = election_state(writer);
     if(rc == election_won) {
         write_attribute(a);
 
     } else if(rc == election_in_progress) {
         crm_trace("Election in progress to determine who will write out %s", a->id);
 
     } else if(peer_writer == NULL) {
         crm_info("Starting an election to determine who will write out %s", a->id);
         election_vote(writer);
 
     } else {
         crm_trace("%s will write out %s, we are in state %d", peer_writer, a->id, rc);
     }
 }
 
 gboolean
 attrd_election_cb(gpointer user_data)
 {
     free(peer_writer);
     peer_writer = strdup(attrd_cluster->uname);
 
     /* Update the peers after an election */
     attrd_peer_sync(NULL, NULL);
 
     /* Update the CIB after an election */
     write_attributes(TRUE, FALSE);
     return FALSE;
 }
 
 
 void
 attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
 {
     if(election_state(writer) == election_won
         && kind == crm_status_nstate
         && safe_str_eq(peer->state, CRM_NODE_MEMBER)) {
 
         attrd_peer_sync(peer, NULL);
 
     } else if(kind == crm_status_nstate
               && safe_str_neq(peer->state, CRM_NODE_MEMBER)) {
 
         attrd_peer_remove(peer->uname, __FUNCTION__);
         if(peer_writer && safe_str_eq(peer->uname, peer_writer)) {
             free(peer_writer);
             peer_writer = NULL;
             crm_notice("Lost attribute writer %s", peer->uname);
         }
 
     } else if(kind == crm_status_processes) {
         if(is_set(peer->processes, crm_proc_cpg)) {
             crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0);
         } else {
             crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0);
         }
     }
 }
 
 static void
 attrd_cib_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     int level = LOG_ERR;
     GHashTableIter iter;
     const char *peer = NULL;
     attribute_value_t *v = NULL;
 
     char *name = user_data;
     attribute_t *a = g_hash_table_lookup(attributes, name);
 
     if(a == NULL) {
         crm_info("Attribute %s no longer exists", name);
         goto done;
     }
 
     a->update = 0;
     if (rc == pcmk_ok && call_id < 0) {
         rc = call_id;
     }
 
     switch (rc) {
         case pcmk_ok:
             level = LOG_INFO;
             last_cib_op_done = call_id;
             break;
         case -pcmk_err_diff_failed:    /* When an attr changes while the CIB is syncing */
         case -ETIME:           /* When an attr changes while there is a DC election */
         case -ENXIO:           /* When an attr changes while the CIB is syncing a
                                 *   newer config from a node that just came up
                                 */
             level = LOG_WARNING;
             break;
     }
 
     do_crm_log(level, "Update %d for %s: %s (%d)", call_id, name, pcmk_strerror(rc), rc);
 
     g_hash_table_iter_init(&iter, a->values);
     while (g_hash_table_iter_next(&iter, (gpointer *) & peer, (gpointer *) & v)) {
         crm_notice("Update %d for %s[%s]=%s: %s (%d)", call_id, a->id, peer, v->requested, pcmk_strerror(rc), rc);
 
         if(rc == pcmk_ok) {
             free(v->stored);
             v->stored = v->requested;
             v->requested = NULL;
 
         } else {
             free(v->requested);
             v->requested = NULL;
             a->changed = TRUE; /* Attempt write out again */
         }
     }
   done:
     free(name);
     if(a && a->changed && election_state(writer) == election_won) {
         write_attribute(a);
     }
 }
 
 void
 write_attributes(bool all, bool peer_discovered)
 {
     GHashTableIter iter;
     attribute_t *a = NULL;
 
     g_hash_table_iter_init(&iter, attributes);
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) {
         if (peer_discovered && a->unknown_peer_uuids) {
             /* a new peer uuid has been discovered, try writing this attribute again. */
             a->changed = TRUE;
         }
 
         if(all || a->changed) {
             write_attribute(a);
         } else {
             crm_debug("Skipping unchanged attribute %s", a->id);
         }
     }
 }
 
 static void
 build_update_element(xmlNode *parent, attribute_t *a, const char *nodeid, const char *value)
 {
     char *set = NULL;
     char *uuid = NULL;
     xmlNode *xml_obj = NULL;
 
     if(a->set) {
         set = g_strdup(a->set);
     } else {
         set = g_strdup_printf("%s-%s", XML_CIB_TAG_STATUS, nodeid);
     }
 
     if(a->uuid) {
         uuid = g_strdup(a->uuid);
     } else {
         int lpc;
         uuid = g_strdup_printf("%s-%s", set, a->id);
 
         /* Minimal attempt at sanitizing automatic IDs */
         for (lpc = 0; uuid[lpc] != 0; lpc++) {
             switch (uuid[lpc]) {
                 case ':':
                     uuid[lpc] = '.';
             }
         }
     }
 
     xml_obj = create_xml_node(parent, XML_CIB_TAG_STATE);
     crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);
 
     xml_obj = create_xml_node(xml_obj, XML_TAG_TRANSIENT_NODEATTRS);
     crm_xml_add(xml_obj, XML_ATTR_ID, nodeid);
 
     xml_obj = create_xml_node(xml_obj, XML_TAG_ATTR_SETS);
     crm_xml_add(xml_obj, XML_ATTR_ID, set);
 
     xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
     crm_xml_add(xml_obj, XML_ATTR_ID, uuid);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, a->id);
 
     if(value) {
         crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, value);
 
     } else {
         crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, "");
         crm_xml_add(xml_obj, "__delete__", XML_NVPAIR_ATTR_VALUE);
     }
 
     g_free(uuid);
     g_free(set);
 }
 
 void
 write_attribute(attribute_t *a)
 {
     int updates = 0;
     xmlNode *xml_top = NULL;
     attribute_value_t *v = NULL;
     GHashTableIter iter;
     enum cib_call_options flags = cib_quorum_override;
 
     if (a == NULL) {
         return;
 
     } else if (the_cib == NULL) {
         crm_info("Write out of %s delayed: cib not connected", a->id);
         return;
 
     } else if(a->update && a->update < last_cib_op_done) {
         crm_info("Write out of %s continuing: update %d considered lost", a->id, a->update);
 
     } else if(a->update) {
         crm_info("Write out of %s delayed: update %d in progress", a->id, a->update);
         return;
 
     } else if(mainloop_timer_running(a->timer)) {
         crm_info("Write out of %s delayed: timer is running", a->id);
         return;
     }
 
     a->changed = FALSE;
     a->unknown_peer_uuids = FALSE;
     xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
 
     g_hash_table_iter_init(&iter, a->values);
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & v)) {
         crm_node_t *peer = crm_get_peer_full(v->nodeid, v->nodename, CRM_GET_PEER_REMOTE|CRM_GET_PEER_CLUSTER);
 
         if(peer && peer->id && v->nodeid == 0) {
             crm_trace("Updating value's nodeid");
             v->nodeid = peer->id;
         }
 
         if (peer == NULL) {
             /* If the user is trying to set an attribute on an unknown peer, ignore it. */
             crm_notice("Update error (peer not found): %s[%s]=%s failed (host=%p)", v->nodename, a->id, v->current, peer);
 
         } else if (peer->uuid == NULL) {
             /* peer is found, but we don't know the uuid yet. Wait until we discover a new uuid before attempting to write */
             a->unknown_peer_uuids = FALSE;
             crm_notice("Update error (unknown peer uuid, retry will be attempted once uuid is discovered): %s[%s]=%s failed (host=%p)", v->nodename, a->id, v->current, peer);
 
         } else {
             crm_debug("Update: %s[%s]=%s (%s %u %u %s)", v->nodename, a->id, v->current, peer->uuid, peer->id, v->nodeid, peer->uname);
             build_update_element(xml_top, a, peer->uuid, v->current);
             updates++;
 
             free(v->requested);
             v->requested = NULL;
 
             if(v->current) {
                 v->requested = strdup(v->current);
 
             } else {
                 /* Older versions don't know about the cib_mixed_update flag
                  * Make sure it goes to the local cib which does
                  */
                 flags |= cib_mixed_update|cib_scope_local;
             }
         }
     }
 
     if(updates) {
         crm_log_xml_trace(xml_top, __FUNCTION__);
 
         a->update = cib_internal_op(the_cib, CIB_OP_MODIFY, NULL, XML_CIB_TAG_STATUS, xml_top, NULL,
                                     flags, a->user);
 
         crm_notice("Sent update %d with %d changes for %s, id=%s, set=%s",
                    a->update, updates, a->id, a->uuid ? a->uuid : "<n/a>", a->set);
 
         the_cib->cmds->register_callback(
             the_cib, a->update, 120, FALSE, strdup(a->id), "attrd_cib_callback", attrd_cib_callback);
     }
 }
diff --git a/include/crm_internal.h b/include/crm_internal.h
index 96b6037d71..ddcb0cada5 100644
--- a/include/crm_internal.h
+++ b/include/crm_internal.h
@@ -1,335 +1,336 @@
 /* crm_internal.h */
 
 /*
  * Copyright (C) 2006 - 2008
  *     Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #ifndef CRM_INTERNAL__H
 #  define CRM_INTERNAL__H
 
 #  include <config.h>
 #  include <portability.h>
 
 #  include <glib.h>
 #  include <stdbool.h>
 #  include <libxml/tree.h>
 
 #  include <crm/lrmd.h>
 #  include <crm/common/logging.h>
 #  include <crm/common/ipcs.h>
 
 /* Dynamic loading of libraries */
 void *find_library_function(void **handle, const char *lib, const char *fn, int fatal);
 void *convert_const_pointer(const void *ptr);
 
 /* For ACLs */
 char *uid2username(uid_t uid);
 void determine_request_user(const char *user, xmlNode * request, const char *field);
 
 #  if ENABLE_ACL
 #    include <string.h>
 static inline gboolean
 is_privileged(const char *user)
 {
     if (user == NULL) {
         return FALSE;
     } else if (strcmp(user, CRM_DAEMON_USER) == 0) {
         return TRUE;
     } else if (strcmp(user, "root") == 0) {
         return TRUE;
     }
     return FALSE;
 }
 #  endif
 
 /* CLI option processing*/
 #  ifdef HAVE_GETOPT_H
 #    include <getopt.h>
 #  else
 #    define no_argument 0
 #    define required_argument 1
 #  endif
 
 #  define pcmk_option_default	0x00000
 #  define pcmk_option_hidden	0x00001
 #  define pcmk_option_paragraph	0x00002
 #  define pcmk_option_example	0x00004
 
 struct crm_option {
     /* Fields from 'struct option' in getopt.h */
     /* name of long option */
     const char *name;
     /*
      * one of no_argument, required_argument, and optional_argument:
      * whether option takes an argument
      */
     int has_arg;
     /* if not NULL, set *flag to val when option found */
     int *flag;
     /* if flag not NULL, value to set *flag to; else return value */
     int val;
 
     /* Custom fields */
     const char *desc;
     long flags;
 };
 
 void crm_set_options(const char *short_options, const char *usage, struct crm_option *long_options,
                      const char *app_desc);
 int crm_get_option(int argc, char **argv, int *index);
 int crm_get_option_long(int argc, char **argv, int *index, const char **longname);
 void crm_help(char cmd, int exit_code);
 
 /* Cluster Option Processing */
 typedef struct pe_cluster_option_s {
     const char *name;
     const char *alt_name;
     const char *type;
     const char *values;
     const char *default_value;
 
      gboolean(*is_valid) (const char *);
 
     const char *description_short;
     const char *description_long;
 
 } pe_cluster_option;
 
 const char *cluster_option(GHashTable * options, gboolean(*validate) (const char *),
                            const char *name, const char *old_name, const char *def_value);
 
 const char *get_cluster_pref(GHashTable * options, pe_cluster_option * option_list, int len,
                              const char *name);
 
 void config_metadata(const char *name, const char *version, const char *desc_short,
                      const char *desc_long, pe_cluster_option * option_list, int len);
 
 void verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len);
 gboolean check_time(const char *value);
 gboolean check_timer(const char *value);
 gboolean check_boolean(const char *value);
 gboolean check_number(const char *value);
 gboolean check_utilization(const char *value);
 
 /* Shared PE/crmd functionality */
 void filter_action_parameters(xmlNode * param_set, const char *version);
 void filter_reload_parameters(xmlNode * param_set, const char *restart_string);
 
 /* Resource operation updates */
 xmlNode *create_operation_update(xmlNode * parent, lrmd_event_data_t * event,
                                  const char *caller_version, int target_rc, const char *origin,
                                  int level);
 
 /* char2score */
 extern int node_score_red;
 extern int node_score_green;
 extern int node_score_yellow;
 extern int node_score_infinity;
 
 /* Assorted convenience functions */
 static inline int
 crm_strlen_zero(const char *s)
 {
     return !s || *s == '\0';
 }
 
 char *add_list_element(char *list, const char *value);
 char *generate_series_filename(const char *directory, const char *series, int sequence,
                                gboolean bzip);
 int get_last_sequence(const char *directory, const char *series);
 void write_last_sequence(const char *directory, const char *series, int sequence, int max);
 
 int crm_pid_active(long pid);
 void crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile);
 gboolean crm_is_writable(const char *dir, const char *file, const char *user, const char *group,
                          gboolean need_both);
 
 char *generate_op_key(const char *rsc_id, const char *op_type, int interval);
 char *generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type);
 char *generate_transition_magic_v202(const char *transition_key, int op_status);
 char *generate_transition_magic(const char *transition_key, int op_status, int op_rc);
 char *generate_transition_key(int action, int transition_id, int target_rc, const char *node);
 
 static inline long long
 crm_clear_bit(const char *function, const char *target, long long word, long long bit)
 {
     long long rc = (word & ~bit);
 
     if (rc == word) {
         /* Unchanged */
     } else if (target) {
         crm_trace("Bit 0x%.8llx for %s cleared by %s", bit, target, function);
     } else {
         crm_trace("Bit 0x%.8llx cleared by %s", bit, function);
     }
 
     return rc;
 }
 
 static inline long long
 crm_set_bit(const char *function, const char *target, long long word, long long bit)
 {
     long long rc = (word | bit);
 
     if (rc == word) {
         /* Unchanged */
     } else if (target) {
         crm_trace("Bit 0x%.8llx for %s set by %s", bit, target, function);
     } else {
         crm_trace("Bit 0x%.8llx set by %s", bit, function);
     }
 
     return rc;
 }
 
 #  define set_bit(word, bit) word = crm_set_bit(__PRETTY_FUNCTION__, NULL, word, bit)
 #  define clear_bit(word, bit) word = crm_clear_bit(__PRETTY_FUNCTION__, NULL, word, bit)
 
 void g_hash_destroy_str(gpointer data);
 
 long long crm_int_helper(const char *text, char **end_text);
 char *crm_concat(const char *prefix, const char *suffix, char join);
 char *generate_hash_key(const char *crm_msg_reference, const char *sys);
 
 bool crm_compress_string(const char *data, int length, int max, char **result,
                          unsigned int *result_len);
 
 /*! remote tcp/tls helper functions */
 typedef struct crm_remote_s crm_remote_t;
 
 int crm_remote_send(crm_remote_t * remote, xmlNode * msg);
 int crm_remote_ready(crm_remote_t * remote, int total_timeout /*ms */ );
 gboolean crm_remote_recv(crm_remote_t * remote, int total_timeout /*ms */ , int *disconnected);
 xmlNode *crm_remote_parse_buffer(crm_remote_t * remote);
 int crm_remote_tcp_connect(const char *host, int port);
 int crm_remote_tcp_connect_async(const char *host, int port, int timeout,       /*ms */
                                  void *userdata, void (*callback) (void *userdata, int sock));
 
 #  ifdef HAVE_GNUTLS_GNUTLS_H
 /*!
  * \internal
  * \brief Initiate the client handshake after establishing the tcp socket.
  * \note This is a blocking function, it will block until the entire handshake
  *       is complete or until the timeout period is reached.
  * \retval 0 success
  * \retval negative, failure
  */
 int crm_initiate_client_tls_handshake(crm_remote_t * remote, int timeout_ms);
 
 /*!
  * \internal
  * \brief Create client or server session for anon DH encryption credentials
  * \param sock, the socket the session will use for transport
  * \param type, GNUTLS_SERVER or GNUTLS_CLIENT
  * \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t
  *
  * \retval gnutls_session_t * on success
  * \retval NULL on failure
  */
 void *crm_create_anon_tls_session(int sock, int type, void *credentials);
 
 /*!
  * \internal
  * \brief Create client or server session for PSK credentials
  * \param sock, the socket the session will use for transport
  * \param type, GNUTLS_SERVER or GNUTLS_CLIENT
  * \param credentials, gnutls_psk_server_credentials_t or gnutls_osk_client_credentials_t
  *
  * \retval gnutls_session_t * on success
  * \retval NULL on failure
  */
 void *create_psk_tls_session(int csock, int type, void *credentials);
 #  endif
 
 #  define REMOTE_MSG_TERMINATOR "\r\n\r\n"
 
 const char *daemon_option(const char *option);
 void set_daemon_option(const char *option, const char *value);
 gboolean daemon_option_enabled(const char *daemon, const char *option);
 void strip_text_nodes(xmlNode * xml);
 
 #  define crm_config_err(fmt...) { crm_config_error = TRUE; crm_err(fmt); }
 #  define crm_config_warn(fmt...) { crm_config_warning = TRUE; crm_warn(fmt); }
 
 #  define attrd_channel		T_ATTRD
 #  define F_ATTRD_KEY		"attr_key"
 #  define F_ATTRD_ATTRIBUTE	"attr_name"
 #  define F_ATTRD_TASK		"task"
 #  define F_ATTRD_VALUE		"attr_value"
 #  define F_ATTRD_SET		"attr_set"
 #  define F_ATTRD_IS_REMOTE	"attr_is_remote"
 #  define F_ATTRD_SECTION	"attr_section"
 #  define F_ATTRD_DAMPEN	"attr_dampening"
 #  define F_ATTRD_IGNORE_LOCALLY "attr_ignore_locally"
 #  define F_ATTRD_HOST		"attr_host"
 #  define F_ATTRD_HOST_ID	"attr_host_id"
 #  define F_ATTRD_USER		"attr_user"
 #  define F_ATTRD_WRITER	"attr_writer"
+#  define F_ATTRD_VERSION	"attr_version"
 
 #  if SUPPORT_COROSYNC
 #    if CS_USES_LIBQB
 #      include <qb/qbipc_common.h>
 #      include <corosync/corotypes.h>
 typedef struct qb_ipc_request_header cs_ipc_header_request_t;
 typedef struct qb_ipc_response_header cs_ipc_header_response_t;
 #    else
 #      include <corosync/corodefs.h>
 #      include <corosync/coroipcc.h>
 #      include <corosync/coroipc_types.h>
 static inline int
 qb_to_cs_error(int a)
 {
     return a;
 }
 
 typedef coroipc_request_header_t cs_ipc_header_request_t;
 typedef coroipc_response_header_t cs_ipc_header_response_t;
 #    endif
 #  else
 typedef struct {
     int size __attribute__ ((aligned(8)));
     int id __attribute__ ((aligned(8)));
 } __attribute__ ((aligned(8))) cs_ipc_header_request_t;
 
 typedef struct {
     int size __attribute__ ((aligned(8)));
     int id __attribute__ ((aligned(8)));
     int error __attribute__ ((aligned(8)));
 } __attribute__ ((aligned(8))) cs_ipc_header_response_t;
 
 #  endif
 
 void
 attrd_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb);
 void
 stonith_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb);
 
 qb_ipcs_service_t *
 crmd_ipc_server_init(struct qb_ipcs_service_handlers *cb);
 
 void cib_ipc_servers_init(qb_ipcs_service_t **ipcs_ro,
         qb_ipcs_service_t **ipcs_rw,
         qb_ipcs_service_t **ipcs_shm,
         struct qb_ipcs_service_handlers *ro_cb,
         struct qb_ipcs_service_handlers *rw_cb);
 
 void cib_ipc_servers_destroy(qb_ipcs_service_t *ipcs_ro,
         qb_ipcs_service_t *ipcs_rw,
         qb_ipcs_service_t *ipcs_shm);
 
 #endif                          /* CRM_INTERNAL__H */