Page MenuHomeClusterLabs Projects
Files F4512329

No OneTemporary
Actions

View Options

diff --git a/server/cpg-virt.c b/server/cpg-virt.c
index 904f1080..fdf59913 100644
--- a/server/cpg-virt.c
+++ b/server/cpg-virt.c
@@ -1,641 +1,653 @@
/*
Copyright Red Hat, Inc. 2017
This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2, or (at your option) any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; see the file COPYING. If not, write to the
Free Software Foundation, Inc., 675 Mass Ave, Cambridge,
MA 02139, USA.
*/
/*
* Author: Ryan McCabe <rmccabe@redhat.com>
*/
#include <config.h>
#include <stdio.h>
#include <simpleconfig.h>
#include <static_map.h>
#include <sys/types.h>
#include <stdint.h>
#include <time.h>
#include <server_plugin.h>
#include <string.h>
#include <syslog.h>
#include <errno.h>
#include <unistd.h>
#include <pthread.h>
#include <corosync/cpg.h>
#include <debug.h>
#include "virt.h"
#include "xvm.h"
#include "cpg.h"
#define NAME "cpg"
#define VERSION "0.1"
#define MAGIC 0x38e93fc2
struct cpg_info {
int magic;
config_object_t *config;
int vp_count;
virConnectPtr *vp;
};
#define VALIDATE(arg) \
do {\
if (!arg || ((struct cpg_info *) arg)->magic != MAGIC) { \
errno = EINVAL;\
return -1; \
} \
} while(0)
static struct cpg_info *cpg_virt_handle = NULL;
static int use_uuid = 0;
pthread_mutex_t local_vm_list_lock = PTHREAD_MUTEX_INITIALIZER;
static virt_list_t *local_vm_list = NULL;
pthread_mutex_t remote_vm_list_lock = PTHREAD_MUTEX_INITIALIZER;
static virt_list_t *remote_vm_list = NULL;
static void cpg_virt_init_libvirt(struct cpg_info *info);
static int
virt_list_update(struct cpg_info *info, virt_list_t **vl, int my_id)
{
virt_list_t *list = NULL;
if (*vl)
vl_free(*vl);
list = vl_get(info->vp, info->vp_count, my_id);
if (!list && (errno == EPIPE || errno == EINVAL)) {
do {
cpg_virt_init_libvirt(info);
} while (info->vp_count == 0);
list = vl_get(info->vp, info->vp_count, my_id);
}
*vl = list;
if (!list)
return -1;
return 0;
}
static void
store_domains(virt_list_t *vl)
{
int i;
if (!vl)
return;
for (i = 0 ; i < vl->vm_count ; i++) {
int ret;
if (!strcmp(DOMAIN0NAME, vl->vm_states[i].v_name))
continue;
ret = cpg_send_vm_state(&vl->vm_states[i]);
if (ret < 0) {
printf("Error storing VM state for %s|%s\n",
vl->vm_states[i].v_name, vl->vm_states[i].v_uuid);
}
}
}
static void
update_local_vms(struct cpg_info *info)
{
uint32_t my_id = 0;
if (!info)
return;
cpg_get_ids(&my_id, NULL);
virt_list_update(info, &local_vm_list, my_id);
store_domains(local_vm_list);
}
static int
do_off(struct cpg_info *info, const char *vm_name)
{
dbg_printf(5, "%s %s\n", __FUNCTION__, vm_name);
return vm_off(info->vp, info->vp_count, vm_name);
}
static int
do_on(struct cpg_info *info, const char *vm_name)
{
dbg_printf(5, "%s %s\n", __FUNCTION__, vm_name);
return vm_on(info->vp, info->vp_count, vm_name);
}
static int
do_reboot(struct cpg_info *info, const char *vm_name)
{
dbg_printf(5, "%s %s\n", __FUNCTION__, vm_name);
return vm_reboot(info->vp, info->vp_count, vm_name);
}
static void
cpg_join_cb(const struct cpg_address *join, size_t joinlen) {
struct cpg_info *info = cpg_virt_handle;
pthread_mutex_lock(&local_vm_list_lock);
update_local_vms(info);
pthread_mutex_unlock(&local_vm_list_lock);
}
static void
cpg_leave_cb(const struct cpg_address *left, size_t leftlen) {
struct cpg_info *info = cpg_virt_handle;
int i;
pthread_mutex_lock(&remote_vm_list_lock);
for (i = 0 ; i < leftlen ; i++) {
dbg_printf(2, "Removing VMs owned by nodeid %u\n", left[i].nodeid);
vl_remove_by_owner(&remote_vm_list, left[i].nodeid);
}
pthread_mutex_unlock(&remote_vm_list_lock);
pthread_mutex_lock(&local_vm_list_lock);
update_local_vms(info);
pthread_mutex_unlock(&local_vm_list_lock);
}
static void
store_cb(void *data, size_t len, uint32_t nodeid, uint32_t seqno)
{
uint32_t my_id;
virt_state_t *vs = (virt_state_t *) data;
struct cpg_info *info = cpg_virt_handle;
cpg_get_ids(&my_id, NULL);
if (nodeid == my_id)
return;
pthread_mutex_lock(&local_vm_list_lock);
if (!local_vm_list)
update_local_vms(info);
pthread_mutex_unlock(&local_vm_list_lock);
pthread_mutex_lock(&remote_vm_list_lock);
vl_update(&remote_vm_list, vs);
pthread_mutex_unlock(&remote_vm_list_lock);
}
/*
** This function must a send reply from at least one node, otherwise
** the requesting fence_virtd will block forever in wait_cpt_reply.
*/
static void
do_real_work(void *data, size_t len, uint32_t nodeid, uint32_t seqno)
{
struct cpg_info *info = cpg_virt_handle;
struct cpg_fence_req *req = data;
struct cpg_fence_req reply;
int reply_code = -1;
virt_state_t *vs = NULL;
int cur_state;
uint32_t cur_owner = 0;
int local = 0;
uint32_t my_id, high_id;
dbg_printf(2, "Request %d for VM %s\n", req->request, req->vm_name);
if (cpg_get_ids(&my_id, &high_id) == -1) {
syslog(LOG_WARNING, "Unable to get CPG IDs");
printf("Should never happen: Can't get CPG node ids - can't proceed\n");
return;
}
memcpy(&reply, req, sizeof(reply));
pthread_mutex_lock(&local_vm_list_lock);
update_local_vms(info);
if (strlen(req->vm_name)) {
if (use_uuid)
vs = vl_find_uuid(local_vm_list, req->vm_name);
else
vs = vl_find_name(local_vm_list, req->vm_name);
if (vs) {
local = 1;
cur_owner = vs->v_state.s_owner;
cur_state = vs->v_state.s_state;
dbg_printf(2, "Found VM %s locally state %d\n",
req->vm_name, cur_state);
}
}
pthread_mutex_unlock(&local_vm_list_lock);
if (vs == NULL) {
pthread_mutex_lock(&remote_vm_list_lock);
if (strlen(req->vm_name)) {
if (use_uuid)
vs = vl_find_uuid(remote_vm_list, req->vm_name);
else
vs = vl_find_name(remote_vm_list, req->vm_name);
if (vs) {
cur_owner = vs->v_state.s_owner;
cur_state = vs->v_state.s_state;
dbg_printf(2, "Found VM %s remotely on %u state %d\n",
req->vm_name, cur_owner, cur_state);
}
}
pthread_mutex_unlock(&remote_vm_list_lock);
}
if (!vs) {
/*
** We know about all domains on all nodes in the CPG group.
** If we didn't find it, and we're high ID, act on the request.
** We can safely assume the VM is OFF because it wasn't found
** on any current members of the CPG group.
*/
if (my_id == high_id) {
if (req->request == FENCE_STATUS)
reply_code = RESP_OFF;
else if (req->request == FENCE_OFF || req->request == FENCE_REBOOT)
reply_code = RESP_SUCCESS;
else
reply_code = 1;
dbg_printf(2, "Acting on request %d for unknown domain %s -> %d\n",
req->request, req->vm_name, reply_code);
goto out;
}
dbg_printf(2, "Not acting on request %d for unknown domain %s\n",
req->request, req->vm_name);
return;
}
if (local) {
if (req->request == FENCE_STATUS) {
/* We already have the status */
if (cur_state == VIR_DOMAIN_SHUTOFF)
reply_code = RESP_OFF;
else
reply_code = RESP_SUCCESS;
} else if (req->request == FENCE_OFF) {
reply_code = do_off(info, req->vm_name);
} else if (req->request == FENCE_ON) {
reply_code = do_on(info, req->vm_name);
} else if (req->request == FENCE_REBOOT) {
reply_code = do_reboot(info, req->vm_name);
} else {
dbg_printf(2, "Not explicitly handling request type %d for %s\n",
req->request, req->vm_name);
reply_code = 0;
}
goto out;
}
/*
** This is a request for a non-local domain that exists on a
** current CPG group member, so that member will see the request
** and act on it. We don't need to do anything.
*/
dbg_printf(2, "Nothing to do for non-local domain %s seq %d owner %u\n",
req->vm_name, seqno, cur_owner);
return;
out:
dbg_printf(2, "[%s] sending reply code seq %d -> %d\n",
req->vm_name, seqno, reply_code);
reply.response = reply_code;
if (cpg_send_reply(&reply, sizeof(reply), nodeid, seqno) < 0) {
dbg_printf(2, "cpg_send_reply failed for %s [%d %d]: %s\n",
req->vm_name, nodeid, seqno, strerror(errno));
}
}
static int
do_request(const char *vm_name, int request, uint32_t seqno)
{
struct cpg_fence_req freq, *frp;
size_t retlen;
uint32_t seq;
int ret;
memset(&freq, 0, sizeof(freq));
if (!vm_name) {
dbg_printf(1, "No VM name\n");
return 1;
}
if (strlen(vm_name) >= sizeof(freq.vm_name)) {
dbg_printf(1, "VM name %s too long\n", vm_name);
return 1;
}
strcpy(freq.vm_name, vm_name);
freq.request = request;
freq.seqno = seqno;
if (cpg_send_req(&freq, sizeof(freq), &seq) != 0) {
dbg_printf(1, "Failed to send request %d for VM %s\n",
freq.request, vm_name);
return 1;
}
dbg_printf(2, "Sent request %d for VM %s got seqno %d\n",
request, vm_name, seq);
if (cpg_wait_reply((void *) &frp, &retlen, seq) != 0) {
dbg_printf(1, "Failed to receive reply seq %d for %s\n", seq, vm_name);
return 1;
}
dbg_printf(2, "Received reply [%d] seq %d for %s\n",
frp->response, seq, vm_name);
ret = frp->response;
free(frp);
return ret;
}
static int
cpg_virt_null(const char *vm_name, void *priv)
{
VALIDATE(priv);
printf("[cpg-virt] Null operation on %s\n", vm_name);
return 1;
}
static int
cpg_virt_off(const char *vm_name, const char *src, uint32_t seqno, void *priv)
{
VALIDATE(priv);
printf("[cpg-virt] OFF operation on %s seq %d\n", vm_name, seqno);
return do_request(vm_name, FENCE_OFF, seqno);
}
static int
cpg_virt_on(const char *vm_name, const char *src, uint32_t seqno, void *priv)
{
VALIDATE(priv);
printf("[cpg-virt] ON operation on %s seq %d\n", vm_name, seqno);
return do_request(vm_name, FENCE_ON, seqno);
}
static int
cpg_virt_devstatus(void *priv)
{
printf("[cpg-virt] Device status\n");
VALIDATE(priv);
return 0;
}
static int
cpg_virt_status(const char *vm_name, void *priv)
{
VALIDATE(priv);
printf("[cpg-virt] STATUS operation on %s\n", vm_name);
return do_request(vm_name, FENCE_STATUS, 0);
}
static int
cpg_virt_reboot(const char *vm_name, const char *src,
uint32_t seqno, void *priv)
{
VALIDATE(priv);
printf("[cpg-virt] REBOOT operation on %s seq %d\n", vm_name, seqno);
return do_request(vm_name, FENCE_REBOOT, 0);
}
static int
cpg_virt_hostlist(hostlist_callback callback, void *arg, void *priv)
{
+ struct cpg_info *info = (struct cpg_info *) priv;
+ int i;
+
VALIDATE(priv);
printf("[cpg-virt] HOSTLIST operation\n");
+ pthread_mutex_lock(&local_vm_list_lock);
+ update_local_vms(info);
+ for (i = 0 ; i < local_vm_list->vm_count ; i++) {
+ callback(local_vm_list->vm_states[i].v_name,
+ local_vm_list->vm_states[i].v_uuid,
+ local_vm_list->vm_states[i].v_state.s_state, arg);
+ }
+ pthread_mutex_unlock(&local_vm_list_lock);
+
return 1;
}
static void
cpg_virt_init_libvirt(struct cpg_info *info) {
config_object_t *config = info->config;
int i = 0;
if (info->vp) {
dbg_printf(2, "Lost libvirtd connection. Reinitializing.\n");
for (i = 0 ; i < info->vp_count ; i++)
virConnectClose(info->vp[i]);
free(info->vp);
info->vp = NULL;
}
info->vp_count = 0;
do {
virConnectPtr vp;
virConnectPtr *vpl = NULL;
char conf_attr[256];
char value[1024];
char *uri;
if (i != 0) {
snprintf(conf_attr, sizeof(conf_attr),
"backends/cpg/@uri%d", i);
} else
snprintf(conf_attr, sizeof(conf_attr), "backends/cpg/@uri");
++i;
if (sc_get(config, conf_attr, value, sizeof(value)) != 0)
break;
uri = value;
vp = virConnectOpen(uri);
if (!vp) {
dbg_printf(1, "[cpg-virt:INIT] Failed to connect to URI: %s\n", uri);
continue;
}
vpl = realloc(info->vp, sizeof(*info->vp) * (info->vp_count + 1));
if (!vpl) {
dbg_printf(1, "[cpg-virt:INIT] Out of memory allocating URI: %s\n",
uri);
virConnectClose(vp);
continue;
}
info->vp = vpl;
info->vp[info->vp_count++] = vp;
if (i > 1)
dbg_printf(1, "[cpg-virt:INIT] Added URI%d %s\n", i - 1, uri);
else
dbg_printf(1, "[cpg_virt:INIT] Added URI %s\n", uri);
} while (1);
}
static int
cpg_virt_init(backend_context_t *c, config_object_t *config)
{
char value[1024];
struct cpg_info *info = NULL;
int ret;
ret = cpg_start(PACKAGE_NAME,
do_real_work, store_cb, cpg_join_cb, cpg_leave_cb);
if (ret < 0)
return -1;
info = calloc(1, sizeof(*info));
if (!info)
return -1;
info->magic = MAGIC;
info->config = config;
#ifdef _MODULE
if (sc_get(config, "fence_virtd/@debug", value, sizeof(value)) == 0)
dset(atoi(value));
#endif
cpg_virt_init_libvirt(info);
/* Naming scheme is no longer a top-level config option.
* However, we retain it here for configuration compatibility with
* versions 0.1.3 and previous.
*/
if (sc_get(config, "fence_virtd/@name_mode",
value, sizeof(value)-1) == 0) {
dbg_printf(1, "Got %s for name_mode\n", value);
if (!strcasecmp(value, "uuid")) {
use_uuid = 1;
} else if (!strcasecmp(value, "name")) {
use_uuid = 0;
} else {
dbg_printf(1, "Unsupported name_mode: %s\n", value);
}
}
if (sc_get(config, "backends/cpg/@name_mode",
value, sizeof(value)-1) == 0)
{
dbg_printf(1, "Got %s for name_mode\n", value);
if (!strcasecmp(value, "uuid")) {
use_uuid = 1;
} else if (!strcasecmp(value, "name")) {
use_uuid = 0;
} else {
dbg_printf(1, "Unsupported name_mode: %s\n", value);
}
}
if (info->vp_count < 1) {
dbg_printf(1, "[cpg_virt:INIT] Could not connect to any hypervisors\n");
cpg_stop();
free(info);
return -1;
}
pthread_mutex_lock(&local_vm_list_lock);
update_local_vms(info);
pthread_mutex_unlock(&local_vm_list_lock);
*c = (void *) info;
cpg_virt_handle = info;
return 0;
}
static int
cpg_virt_shutdown(backend_context_t c)
{
struct cpg_info *info = (struct cpg_info *)c;
int i = 0;
int ret = 0;
VALIDATE(info);
info->magic = 0;
cpg_stop();
for (i = 0 ; i < info->vp_count ; i++) {
if (virConnectClose(info->vp[i]) < 0)
ret = -errno;
}
+
free(info->vp);
free(info);
-
return ret;
}
static fence_callbacks_t cpg_callbacks = {
.null = cpg_virt_null,
.off = cpg_virt_off,
.on = cpg_virt_on,
.reboot = cpg_virt_reboot,
.status = cpg_virt_status,
.devstatus = cpg_virt_devstatus,
.hostlist = cpg_virt_hostlist
};
static backend_plugin_t cpg_virt_plugin = {
.name = NAME,
.version = VERSION,
.callbacks = &cpg_callbacks,
.init = cpg_virt_init,
.cleanup = cpg_virt_shutdown,
};
#ifdef _MODULE
double
BACKEND_VER_SYM(void)
{
return PLUGIN_VERSION_BACKEND;
}
const backend_plugin_t *
BACKEND_INFO_SYM(void)
{
return &cpg_virt_plugin;
}
#else
static void __attribute__((constructor))
cpg_register_plugin(void)
{
plugin_reg_backend(&cpg_virt_plugin);
}
#endif

File Metadata

Mime Type
text/x-diff
Expires
Wed, Jun 25, 5:07 AM (1 d, 14 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1952227
Default Alt Text
(15 KB)

Event Timeline