Page MenuHomeClusterLabs Projects
Files F3153261

No OneTemporary
Actions

View Options

diff --git a/exec/totemcrypto.c b/exec/totemcrypto.c
index 9e70fec2..9f8f083c 100644
--- a/exec/totemcrypto.c
+++ b/exec/totemcrypto.c
@@ -1,969 +1,975 @@
/*
* Copyright (c) 2006-2012 Red Hat, Inc.
*
* All rights reserved.
*
* Author: Steven Dake (sdake@redhat.com)
* Christine Caulfield (ccaulfie@redhat.com)
* Jan Friesse (jfriesse@redhat.com)
* Fabio M. Di Nitto (fdinitto@redhat.com)
*
* This software licensed under BSD license, the text of which follows:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* - Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* - Neither the name of the MontaVista Software, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <nss.h>
#include <pk11pub.h>
#include <pkcs11.h>
#include <prerror.h>
#include <blapit.h>
#include <hasht.h>
#define LOGSYS_UTILS_ONLY 1
#include <corosync/logsys.h>
#include <corosync/totem/totem.h>
#include "totemcrypto.h"
/*
* define onwire crypto header
*/
struct crypto_config_header {
uint8_t crypto_cipher_type;
uint8_t crypto_hash_type;
uint8_t __pad0;
uint8_t __pad1;
} __attribute__((packed));
/*
* crypto definitions and conversion tables
*/
#define SALT_SIZE 16
/*
* This are defined in new NSS. For older one, we will define our own
*/
#ifndef AES_256_KEY_LENGTH
#define AES_256_KEY_LENGTH 32
#endif
#ifndef AES_192_KEY_LENGTH
#define AES_192_KEY_LENGTH 24
#endif
#ifndef AES_128_KEY_LENGTH
#define AES_128_KEY_LENGTH 16
#endif
/*
* while CRYPTO_CIPHER_TYPE_2_X are not a real cipher at all,
* we still allocate a value for them because we use crypto_crypt_t
* internally and we don't want overlaps
*/
enum crypto_crypt_t {
CRYPTO_CIPHER_TYPE_NONE = 0,
CRYPTO_CIPHER_TYPE_AES256 = 1,
CRYPTO_CIPHER_TYPE_AES192 = 2,
CRYPTO_CIPHER_TYPE_AES128 = 3,
CRYPTO_CIPHER_TYPE_3DES = 4,
CRYPTO_CIPHER_TYPE_2_3 = UINT8_MAX - 1,
CRYPTO_CIPHER_TYPE_2_2 = UINT8_MAX
};
CK_MECHANISM_TYPE cipher_to_nss[] = {
0, /* CRYPTO_CIPHER_TYPE_NONE */
CKM_AES_CBC_PAD, /* CRYPTO_CIPHER_TYPE_AES256 */
CKM_AES_CBC_PAD, /* CRYPTO_CIPHER_TYPE_AES192 */
CKM_AES_CBC_PAD, /* CRYPTO_CIPHER_TYPE_AES128 */
CKM_DES3_CBC_PAD /* CRYPTO_CIPHER_TYPE_3DES */
};
size_t cipher_key_len[] = {
0, /* CRYPTO_CIPHER_TYPE_NONE */
AES_256_KEY_LENGTH, /* CRYPTO_CIPHER_TYPE_AES256 */
AES_192_KEY_LENGTH, /* CRYPTO_CIPHER_TYPE_AES192 */
AES_128_KEY_LENGTH, /* CRYPTO_CIPHER_TYPE_AES128 */
24 /* CRYPTO_CIPHER_TYPE_3DES - no magic in nss headers */
};
size_t cypher_block_len[] = {
0, /* CRYPTO_CIPHER_TYPE_NONE */
AES_BLOCK_SIZE, /* CRYPTO_CIPHER_TYPE_AES256 */
AES_BLOCK_SIZE, /* CRYPTO_CIPHER_TYPE_AES192 */
AES_BLOCK_SIZE, /* CRYPTO_CIPHER_TYPE_AES128 */
0 /* CRYPTO_CIPHER_TYPE_3DES */
};
/*
* hash definitions and conversion tables
*/
/*
* while CRYPTO_HASH_TYPE_2_X are not a real hash mechanism at all,
* we still allocate a value for them because we use crypto_hash_t
* internally and we don't want overlaps
*/
enum crypto_hash_t {
CRYPTO_HASH_TYPE_NONE = 0,
CRYPTO_HASH_TYPE_MD5 = 1,
CRYPTO_HASH_TYPE_SHA1 = 2,
CRYPTO_HASH_TYPE_SHA256 = 3,
CRYPTO_HASH_TYPE_SHA384 = 4,
CRYPTO_HASH_TYPE_SHA512 = 5,
CRYPTO_HASH_TYPE_2_3 = UINT8_MAX - 1,
CRYPTO_HASH_TYPE_2_2 = UINT8_MAX
};
CK_MECHANISM_TYPE hash_to_nss[] = {
0, /* CRYPTO_HASH_TYPE_NONE */
CKM_MD5_HMAC, /* CRYPTO_HASH_TYPE_MD5 */
CKM_SHA_1_HMAC, /* CRYPTO_HASH_TYPE_SHA1 */
CKM_SHA256_HMAC, /* CRYPTO_HASH_TYPE_SHA256 */
CKM_SHA384_HMAC, /* CRYPTO_HASH_TYPE_SHA384 */
CKM_SHA512_HMAC /* CRYPTO_HASH_TYPE_SHA512 */
};
size_t hash_len[] = {
0, /* CRYPTO_HASH_TYPE_NONE */
MD5_LENGTH, /* CRYPTO_HASH_TYPE_MD5 */
SHA1_LENGTH, /* CRYPTO_HASH_TYPE_SHA1 */
SHA256_LENGTH, /* CRYPTO_HASH_TYPE_SHA256 */
SHA384_LENGTH, /* CRYPTO_HASH_TYPE_SHA384 */
SHA512_LENGTH /* CRYPTO_HASH_TYPE_SHA512 */
};
size_t hash_block_len[] = {
0, /* CRYPTO_HASH_TYPE_NONE */
MD5_BLOCK_LENGTH, /* CRYPTO_HASH_TYPE_MD5 */
SHA1_BLOCK_LENGTH, /* CRYPTO_HASH_TYPE_SHA1 */
SHA256_BLOCK_LENGTH, /* CRYPTO_HASH_TYPE_SHA256 */
SHA384_BLOCK_LENGTH, /* CRYPTO_HASH_TYPE_SHA384 */
SHA512_BLOCK_LENGTH /* CRYPTO_HASH_TYPE_SHA512 */
};
struct crypto_instance {
PK11SymKey *nss_sym_key;
PK11SymKey *nss_sym_key_sign;
unsigned char private_key[1024];
unsigned int private_key_len;
enum crypto_crypt_t crypto_cipher_type;
enum crypto_hash_t crypto_hash_type;
unsigned int crypto_header_size;
void (*log_printf_func) (
int level,
int subsys,
const char *function,
const char *file,
int line,
const char *format,
...)__attribute__((format(printf, 6, 7)));
int log_level_security;
int log_level_notice;
int log_level_error;
int log_subsys_id;
};
#define log_printf(level, format, args...) \
do { \
instance->log_printf_func ( \
level, instance->log_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
(const char *)format, ##args); \
} while (0);
enum sym_key_type {
SYM_KEY_TYPE_CRYPT,
SYM_KEY_TYPE_HASH
};
#define MAX_WRAPPED_KEY_LEN 128
/*
* crypt/decrypt functions
*/
static int string_to_crypto_cipher_type(const char* crypto_cipher_type)
{
if (strcmp(crypto_cipher_type, "none") == 0) {
return CRYPTO_CIPHER_TYPE_NONE;
} else if (strcmp(crypto_cipher_type, "aes256") == 0) {
return CRYPTO_CIPHER_TYPE_AES256;
} else if (strcmp(crypto_cipher_type, "aes192") == 0) {
return CRYPTO_CIPHER_TYPE_AES192;
} else if (strcmp(crypto_cipher_type, "aes128") == 0) {
return CRYPTO_CIPHER_TYPE_AES128;
} else if (strcmp(crypto_cipher_type, "3des") == 0) {
return CRYPTO_CIPHER_TYPE_3DES;
}
return CRYPTO_CIPHER_TYPE_AES256;
}
static PK11SymKey *import_symmetric_key(struct crypto_instance *instance, enum sym_key_type key_type)
{
SECItem key_item;
PK11SlotInfo *slot;
PK11SymKey *res_key;
CK_MECHANISM_TYPE cipher;
CK_ATTRIBUTE_TYPE operation;
CK_MECHANISM_TYPE wrap_mechanism;
int wrap_key_len;
PK11SymKey *wrap_key;
PK11Context *wrap_key_crypt_context;
SECItem tmp_sec_item;
SECItem wrapped_key;
int wrapped_key_len;
unsigned char wrapped_key_data[MAX_WRAPPED_KEY_LEN];
int case_processed;
memset(&key_item, 0, sizeof(key_item));
slot = NULL;
wrap_key = NULL;
res_key = NULL;
wrap_key_crypt_context = NULL;
key_item.type = siBuffer;
key_item.data = instance->private_key;
case_processed = 0;
switch (key_type) {
case SYM_KEY_TYPE_CRYPT:
key_item.len = cipher_key_len[instance->crypto_cipher_type];
cipher = cipher_to_nss[instance->crypto_cipher_type];
operation = CKA_ENCRYPT|CKA_DECRYPT;
case_processed = 1;
break;
case SYM_KEY_TYPE_HASH:
key_item.len = instance->private_key_len;
cipher = hash_to_nss[instance->crypto_hash_type];
operation = CKA_SIGN;
case_processed = 1;
break;
/*
* Default is not defined intentionally. Compiler shows warning when
* new key_type is added
*/
}
if (!case_processed) {
log_printf(instance->log_level_error, "Unknown key_type");
goto exit_res_key;
}
slot = PK11_GetBestSlot(cipher, NULL);
if (slot == NULL) {
log_printf(instance->log_level_security, "Unable to find security slot (%d): %s",
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto exit_res_key;
}
/*
* Without FIPS it would be possible to just use
* res_key = PK11_ImportSymKey(slot, cipher, PK11_OriginUnwrap, operation, &key_item, NULL);
* with FIPS NSS Level 2 certification has to be "workarounded" (so it becomes Level 1) by using
* following method:
* 1. Generate wrap key
* 2. Encrypt authkey with wrap key
* 3. Unwrap encrypted authkey using wrap key
*/
/*
* Generate wrapping key
*/
wrap_mechanism = PK11_GetBestWrapMechanism(slot);
wrap_key_len = PK11_GetBestKeyLength(slot, wrap_mechanism);
wrap_key = PK11_KeyGen(slot, wrap_mechanism, NULL, wrap_key_len, NULL);
if (wrap_key == NULL) {
log_printf(instance->log_level_security, "Unable to generate wrapping key (%d): %s",
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto exit_res_key;
}
/*
* Encrypt authkey with wrapping key
*/
/*
* Initialization of IV is not needed because PK11_GetBestWrapMechanism should return ECB mode
*/
memset(&tmp_sec_item, 0, sizeof(tmp_sec_item));
wrap_key_crypt_context = PK11_CreateContextBySymKey(wrap_mechanism, CKA_ENCRYPT,
wrap_key, &tmp_sec_item);
if (wrap_key_crypt_context == NULL) {
log_printf(instance->log_level_security, "Unable to create encrypt context (%d): %s",
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto exit_res_key;
}
wrapped_key_len = (int)sizeof(wrapped_key_data);
if (PK11_CipherOp(wrap_key_crypt_context, wrapped_key_data, &wrapped_key_len,
sizeof(wrapped_key_data), key_item.data, key_item.len) != SECSuccess) {
log_printf(instance->log_level_security, "Unable to encrypt authkey (%d): %s",
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto exit_res_key;
}
if (PK11_Finalize(wrap_key_crypt_context) != SECSuccess) {
log_printf(instance->log_level_security, "Unable to finalize encryption of authkey (%d): %s",
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto exit_res_key;
}
/*
* Finally unwrap sym key
*/
memset(&tmp_sec_item, 0, sizeof(tmp_sec_item));
wrapped_key.data = wrapped_key_data;
wrapped_key.len = wrapped_key_len;
res_key = PK11_UnwrapSymKey(wrap_key, wrap_mechanism, &tmp_sec_item, &wrapped_key,
cipher, operation, key_item.len);
if (res_key == NULL) {
log_printf(instance->log_level_security, "Failure to import key into NSS (%d): %s",
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto exit_res_key;
}
exit_res_key:
if (wrap_key_crypt_context != NULL) {
PK11_DestroyContext(wrap_key_crypt_context, PR_TRUE);
}
if (wrap_key != NULL) {
PK11_FreeSymKey(wrap_key);
}
if (slot != NULL) {
PK11_FreeSlot(slot);
}
return (res_key);
}
static int init_nss_crypto(struct crypto_instance *instance)
{
if (!cipher_to_nss[instance->crypto_cipher_type]) {
return 0;
}
instance->nss_sym_key = import_symmetric_key(instance, SYM_KEY_TYPE_CRYPT);
if (instance->nss_sym_key == NULL) {
return -1;
}
return 0;
}
static int encrypt_nss(
struct crypto_instance *instance,
const unsigned char *buf_in,
const size_t buf_in_len,
unsigned char *buf_out,
size_t *buf_out_len)
{
PK11Context* crypt_context = NULL;
SECItem crypt_param;
SECItem *nss_sec_param = NULL;
int tmp1_outlen = 0;
unsigned int tmp2_outlen = 0;
unsigned char *salt = buf_out;
unsigned char *data = buf_out + SALT_SIZE;
int err = -1;
if (!cipher_to_nss[instance->crypto_cipher_type]) {
memcpy(buf_out, buf_in, buf_in_len);
*buf_out_len = buf_in_len;
return 0;
}
if (PK11_GenerateRandom (salt, SALT_SIZE) != SECSuccess) {
log_printf(instance->log_level_security,
"Failure to generate a random number %d",
PR_GetError());
goto out;
}
crypt_param.type = siBuffer;
crypt_param.data = salt;
crypt_param.len = SALT_SIZE;
nss_sec_param = PK11_ParamFromIV (cipher_to_nss[instance->crypto_cipher_type],
&crypt_param);
if (nss_sec_param == NULL) {
log_printf(instance->log_level_security,
"Failure to set up PKCS11 param (err %d)",
PR_GetError());
goto out;
}
/*
* Create cipher context for encryption
*/
crypt_context = PK11_CreateContextBySymKey (cipher_to_nss[instance->crypto_cipher_type],
CKA_ENCRYPT,
instance->nss_sym_key,
nss_sec_param);
if (!crypt_context) {
log_printf(instance->log_level_security,
"PK11_CreateContext failed (encrypt) crypt_type=%d (%d): %s",
(int)cipher_to_nss[instance->crypto_cipher_type],
PR_GetError(), PR_ErrorToString(PR_GetError(), PR_LANGUAGE_I_DEFAULT));
goto out;
}
if (PK11_CipherOp(crypt_context, data,
&tmp1_outlen,
FRAME_SIZE_MAX - instance->crypto_header_size,
(unsigned char *)buf_in, buf_in_len) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_CipherOp failed (encrypt) crypt_type=%d (err %d)",
(int)cipher_to_nss[instance->crypto_cipher_type],
PR_GetError());
goto out;
}
if (PK11_DigestFinal(crypt_context, data + tmp1_outlen,
&tmp2_outlen, FRAME_SIZE_MAX - tmp1_outlen) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_DigestFinal failed (encrypt) crypt_type=%d (err %d)",
(int)cipher_to_nss[instance->crypto_cipher_type],
PR_GetError());
goto out;
}
*buf_out_len = tmp1_outlen + tmp2_outlen + SALT_SIZE;
err = 0;
out:
if (crypt_context) {
PK11_DestroyContext(crypt_context, PR_TRUE);
}
if (nss_sec_param) {
SECITEM_FreeItem(nss_sec_param, PR_TRUE);
}
return err;
}
static int decrypt_nss (
struct crypto_instance *instance,
unsigned char *buf,
int *buf_len)
{
PK11Context* decrypt_context = NULL;
SECItem decrypt_param;
int tmp1_outlen = 0;
unsigned int tmp2_outlen = 0;
unsigned char *salt = buf;
unsigned char *data = salt + SALT_SIZE;
int datalen = *buf_len - SALT_SIZE;
unsigned char outbuf[FRAME_SIZE_MAX];
int outbuf_len;
int err = -1;
if (!cipher_to_nss[instance->crypto_cipher_type]) {
return 0;
}
/* Create cipher context for decryption */
decrypt_param.type = siBuffer;
decrypt_param.data = salt;
decrypt_param.len = SALT_SIZE;
decrypt_context = PK11_CreateContextBySymKey(cipher_to_nss[instance->crypto_cipher_type],
CKA_DECRYPT,
instance->nss_sym_key, &decrypt_param);
if (!decrypt_context) {
log_printf(instance->log_level_security,
"PK11_CreateContext (decrypt) failed (err %d)",
PR_GetError());
goto out;
}
if (PK11_CipherOp(decrypt_context, outbuf, &tmp1_outlen,
sizeof(outbuf), data, datalen) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_CipherOp (decrypt) failed (err %d)",
PR_GetError());
goto out;
}
if (PK11_DigestFinal(decrypt_context, outbuf + tmp1_outlen, &tmp2_outlen,
sizeof(outbuf) - tmp1_outlen) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_DigestFinal (decrypt) failed (err %d)",
PR_GetError());
goto out;
}
outbuf_len = tmp1_outlen + tmp2_outlen;
memset(buf, 0, *buf_len);
memcpy(buf, outbuf, outbuf_len);
*buf_len = outbuf_len;
err = 0;
out:
if (decrypt_context) {
PK11_DestroyContext(decrypt_context, PR_TRUE);
}
return err;
}
/*
* hash/hmac/digest functions
*/
static int string_to_crypto_hash_type(const char* crypto_hash_type)
{
if (strcmp(crypto_hash_type, "none") == 0) {
return CRYPTO_HASH_TYPE_NONE;
} else if (strcmp(crypto_hash_type, "md5") == 0) {
return CRYPTO_HASH_TYPE_MD5;
} else if (strcmp(crypto_hash_type, "sha1") == 0) {
return CRYPTO_HASH_TYPE_SHA1;
} else if (strcmp(crypto_hash_type, "sha256") == 0) {
return CRYPTO_HASH_TYPE_SHA256;
} else if (strcmp(crypto_hash_type, "sha384") == 0) {
return CRYPTO_HASH_TYPE_SHA384;
} else if (strcmp(crypto_hash_type, "sha512") == 0) {
return CRYPTO_HASH_TYPE_SHA512;
}
return CRYPTO_HASH_TYPE_SHA1;
}
static int init_nss_hash(struct crypto_instance *instance)
{
if (!hash_to_nss[instance->crypto_hash_type]) {
return 0;
}
instance->nss_sym_key_sign = import_symmetric_key(instance, SYM_KEY_TYPE_HASH);
if (instance->nss_sym_key_sign == NULL) {
return -1;
}
return 0;
}
static int calculate_nss_hash(
struct crypto_instance *instance,
const unsigned char *buf,
const size_t buf_len,
unsigned char *hash)
{
PK11Context* hash_context = NULL;
SECItem hash_param;
unsigned int hash_tmp_outlen = 0;
unsigned char hash_block[hash_block_len[instance->crypto_hash_type]];
int err = -1;
/* Now do the digest */
hash_param.type = siBuffer;
hash_param.data = 0;
hash_param.len = 0;
hash_context = PK11_CreateContextBySymKey(hash_to_nss[instance->crypto_hash_type],
CKA_SIGN,
instance->nss_sym_key_sign,
&hash_param);
if (!hash_context) {
log_printf(instance->log_level_security,
"PK11_CreateContext failed (hash) hash_type=%d (err %d)",
(int)hash_to_nss[instance->crypto_hash_type],
PR_GetError());
goto out;
}
if (PK11_DigestBegin(hash_context) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_DigestBegin failed (hash) hash_type=%d (err %d)",
(int)hash_to_nss[instance->crypto_hash_type],
PR_GetError());
goto out;
}
if (PK11_DigestOp(hash_context,
buf,
buf_len) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_DigestOp failed (hash) hash_type=%d (err %d)",
(int)hash_to_nss[instance->crypto_hash_type],
PR_GetError());
goto out;
}
if (PK11_DigestFinal(hash_context,
hash_block,
&hash_tmp_outlen,
hash_block_len[instance->crypto_hash_type]) != SECSuccess) {
log_printf(instance->log_level_security,
"PK11_DigestFinale failed (hash) hash_type=%d (err %d)",
(int)hash_to_nss[instance->crypto_hash_type],
PR_GetError());
goto out;
}
memcpy(hash, hash_block, hash_len[instance->crypto_hash_type]);
err = 0;
out:
if (hash_context) {
PK11_DestroyContext(hash_context, PR_TRUE);
}
return err;
}
/*
* global/glue nss functions
*/
static int init_nss_db(struct crypto_instance *instance)
{
if ((!cipher_to_nss[instance->crypto_cipher_type]) &&
(!hash_to_nss[instance->crypto_hash_type])) {
return 0;
}
if (NSS_NoDB_Init(".") != SECSuccess) {
log_printf(instance->log_level_security, "NSS DB initialization failed (err %d)",
PR_GetError());
return -1;
}
return 0;
}
static int init_nss(struct crypto_instance *instance,
const char *crypto_cipher_type,
const char *crypto_hash_type)
{
log_printf(instance->log_level_notice,
"Initializing transmit/receive security (NSS) crypto: %s hash: %s",
crypto_cipher_type, crypto_hash_type);
if (init_nss_db(instance) < 0) {
return -1;
}
if (init_nss_crypto(instance) < 0) {
return -1;
}
if (init_nss_hash(instance) < 0) {
return -1;
}
return 0;
}
static int encrypt_and_sign_nss_2_3 (
struct crypto_instance *instance,
const unsigned char *buf_in,
const size_t buf_in_len,
unsigned char *buf_out,
size_t *buf_out_len)
{
if (encrypt_nss(instance,
buf_in, buf_in_len,
buf_out + sizeof(struct crypto_config_header), buf_out_len) < 0) {
return -1;
}
*buf_out_len += sizeof(struct crypto_config_header);
if (hash_to_nss[instance->crypto_hash_type]) {
if (calculate_nss_hash(instance, buf_out, *buf_out_len, buf_out + *buf_out_len) < 0) {
return -1;
}
*buf_out_len += hash_len[instance->crypto_hash_type];
}
return 0;
}
static int authenticate_nss_2_3 (
struct crypto_instance *instance,
unsigned char *buf,
int *buf_len)
{
if (hash_to_nss[instance->crypto_hash_type]) {
unsigned char tmp_hash[hash_len[instance->crypto_hash_type]];
int datalen = *buf_len - hash_len[instance->crypto_hash_type];
if (*buf_len <= hash_len[instance->crypto_hash_type]) {
log_printf(instance->log_level_security, "Received message is too short... ignoring");
return -1;
}
if (calculate_nss_hash(instance, buf, datalen, tmp_hash) < 0) {
return -1;
}
if (memcmp(tmp_hash, buf + datalen, hash_len[instance->crypto_hash_type]) != 0) {
log_printf(instance->log_level_error, "Digest does not match");
return -1;
}
*buf_len = datalen;
}
return 0;
}
static int decrypt_nss_2_3 (
struct crypto_instance *instance,
unsigned char *buf,
int *buf_len)
{
*buf_len -= sizeof(struct crypto_config_header);
if (decrypt_nss(instance, buf + sizeof(struct crypto_config_header), buf_len) < 0) {
return -1;
}
return 0;
}
/*
* exported API
*/
size_t crypto_sec_header_size(
const char *crypto_cipher_type,
const char *crypto_hash_type)
{
int crypto_cipher = string_to_crypto_cipher_type(crypto_cipher_type);
int crypto_hash = string_to_crypto_hash_type(crypto_hash_type);
size_t hdr_size = 0;
int block_size = 0;
hdr_size = sizeof(struct crypto_config_header);
if (crypto_hash) {
hdr_size += hash_len[crypto_hash];
}
if (crypto_cipher) {
hdr_size += SALT_SIZE;
if (cypher_block_len[crypto_cipher]) {
block_size = cypher_block_len[crypto_cipher];
} else {
block_size = PK11_GetBlockSize(crypto_cipher, NULL);
if (block_size < 0) {
/*
* failsafe. we can potentially lose up to 63
* byte per packet, but better than fragmenting
*/
block_size = 64;
}
}
hdr_size += (block_size * 2);
}
return hdr_size;
}
+size_t crypto_get_current_sec_header_size(
+ const struct crypto_instance *instance)
+{
+
+ return (instance->crypto_header_size);
+}
+
/*
* 2.0 packet format:
* crypto_cipher_type | crypto_hash_type | __pad0 | __pad1 | hash | salt | data
* only data is encrypted, hash only covers salt + data
*
* 2.2/2.3 packet format
* fake_crypto_cipher_type | fake_crypto_hash_type | __pad0 | __pad1 | salt | data | hash
* only data is encrypted, hash covers the whole packet
*
* we need to leave fake_* unencrypted for older versions of corosync to reject the packets,
* we need to leave __pad0|1 unencrypted for performance reasons (saves at least 2 memcpy and
* and extra buffer but values are hashed and verified.
*/
int crypto_encrypt_and_sign (
struct crypto_instance *instance,
const unsigned char *buf_in,
const size_t buf_in_len,
unsigned char *buf_out,
size_t *buf_out_len)
{
struct crypto_config_header *cch = (struct crypto_config_header *)buf_out;
int err;
cch->crypto_cipher_type = CRYPTO_CIPHER_TYPE_2_3;
cch->crypto_hash_type = CRYPTO_HASH_TYPE_2_3;
cch->__pad0 = 0;
cch->__pad1 = 0;
err = encrypt_and_sign_nss_2_3(instance,
buf_in, buf_in_len,
buf_out, buf_out_len);
return err;
}
int crypto_authenticate_and_decrypt (struct crypto_instance *instance,
unsigned char *buf,
int *buf_len)
{
struct crypto_config_header *cch = (struct crypto_config_header *)buf;
const char *guessed_str;
if (*buf_len <= sizeof(struct crypto_config_header)) {
log_printf(instance->log_level_security, "Received message is too short... ignoring");
return (-1);
}
if (cch->crypto_cipher_type != CRYPTO_CIPHER_TYPE_2_3) {
guessed_str = NULL;
if ((cch->crypto_cipher_type == 0xC0 && cch->crypto_hash_type == 0x70) ||
(cch->crypto_cipher_type == 0x70 && cch->crypto_hash_type == 0xC0)) {
guessed_str = "Corosync 3.x";
} else if (cch->crypto_cipher_type == CRYPTO_CIPHER_TYPE_2_2) {
guessed_str = "Corosync 2.2";
} else if (cch->crypto_cipher_type == 0x01) {
guessed_str = "unencrypted Kronosnet";
} else if (cch->crypto_cipher_type >= 0 && cch->crypto_cipher_type <= 5) {
guessed_str = "unencrypted Corosync 2.0/2.1/1.x/OpenAIS";
} else {
guessed_str = "encrypted Kronosnet/Corosync 2.0/2.1/1.x/OpenAIS or unknown";
}
log_printf(instance->log_level_security,
"Unsupported incoming packet (probably sent by %s). Rejecting",
guessed_str);
return -1;
}
if (cch->crypto_hash_type != CRYPTO_HASH_TYPE_2_3) {
log_printf(instance->log_level_security,
"Incoming packet has different hash type. Rejecting");
return -1;
}
/*
* authenticate packet first
*/
if (authenticate_nss_2_3(instance, buf, buf_len) != 0) {
return -1;
}
/*
* now we can "trust" the padding bytes/future features
*/
if ((cch->__pad0 != 0) || (cch->__pad1 != 0)) {
log_printf(instance->log_level_security,
"Incoming packet appears to have features not supported by this version of corosync. Rejecting");
return -1;
}
/*
* decrypt
*/
-
if (decrypt_nss_2_3(instance, buf, buf_len) != 0) {
return -1;
}
/*
* invalidate config header and kill it
*/
cch = NULL;
memmove(buf, buf + sizeof(struct crypto_config_header), *buf_len);
return 0;
}
struct crypto_instance *crypto_init(
const unsigned char *private_key,
unsigned int private_key_len,
const char *crypto_cipher_type,
const char *crypto_hash_type,
void (*log_printf_func) (
int level,
int subsys,
const char *function,
const char *file,
int line,
const char *format,
...)__attribute__((format(printf, 6, 7))),
int log_level_security,
int log_level_notice,
int log_level_error,
int log_subsys_id)
{
struct crypto_instance *instance;
instance = malloc(sizeof(*instance));
if (instance == NULL) {
return (NULL);
}
memset(instance, 0, sizeof(struct crypto_instance));
memcpy(instance->private_key, private_key, private_key_len);
instance->private_key_len = private_key_len;
instance->crypto_cipher_type = string_to_crypto_cipher_type(crypto_cipher_type);
instance->crypto_hash_type = string_to_crypto_hash_type(crypto_hash_type);
instance->crypto_header_size = crypto_sec_header_size(crypto_cipher_type, crypto_hash_type);
instance->log_printf_func = log_printf_func;
instance->log_level_security = log_level_security;
instance->log_level_notice = log_level_notice;
instance->log_level_error = log_level_error;
instance->log_subsys_id = log_subsys_id;
if (init_nss(instance, crypto_cipher_type, crypto_hash_type) < 0) {
free(instance);
return(NULL);
}
return (instance);
}
diff --git a/exec/totemcrypto.h b/exec/totemcrypto.h
index 7c06c391..b6941055 100644
--- a/exec/totemcrypto.h
+++ b/exec/totemcrypto.h
@@ -1,77 +1,80 @@
/*
* Copyright (c) 2006-2012 Red Hat, Inc.
*
* All rights reserved.
*
* Author: Steven Dake (sdake@redhat.com)
* Christine Caulfield (ccaulfie@redhat.com)
* Jan Friesse (jfriesse@redhat.com)
*
* This software licensed under BSD license, the text of which follows:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* - Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* - Neither the name of the MontaVista Software, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef TOTEMCRYPTO_H_DEFINED
#define TOTEMCRYPTO_H_DEFINED
#include <sys/types.h>
struct crypto_instance;
extern size_t crypto_sec_header_size(
const char *crypto_cipher_type,
const char *crypto_hash_type);
+extern size_t crypto_get_current_sec_header_size(
+ const struct crypto_instance *instance);
+
extern int crypto_authenticate_and_decrypt (
struct crypto_instance *instance,
unsigned char *buf,
int *buf_len);
extern int crypto_encrypt_and_sign (
struct crypto_instance *instance,
const unsigned char *buf_in,
const size_t buf_in_len,
unsigned char *buf_out,
size_t *buf_out_len);
extern struct crypto_instance *crypto_init(
const unsigned char *private_key,
unsigned int private_key_len,
const char *crypto_cipher_type,
const char *crypto_hash_type,
void (*log_printf_func) (
int level,
int subsys,
const char *function,
const char *file,
int line,
const char *format,
...)__attribute__((format(printf, 6, 7))),
int log_level_security,
int log_level_notice,
int log_level_error,
int log_subsys_id);
#endif /* TOTEMCRYPTO_H_DEFINED */
diff --git a/exec/totemudp.c b/exec/totemudp.c
index 40e99f93..4b644aee 100644
--- a/exec/totemudp.c
+++ b/exec/totemudp.c
@@ -1,1495 +1,1507 @@
/*
* Copyright (c) 2005 MontaVista Software, Inc.
* Copyright (c) 2006-2012 Red Hat, Inc.
*
* All rights reserved.
*
* Author: Steven Dake (sdake@redhat.com)
* This software licensed under BSD license, the text of which follows:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* - Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* - Neither the name of the MontaVista Software, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <config.h>
#include <assert.h>
#include <pthread.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netdb.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <sys/param.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <sched.h>
#include <time.h>
#include <sys/time.h>
#include <sys/poll.h>
#include <sys/uio.h>
#include <limits.h>
#include <corosync/sq.h>
#include <corosync/swab.h>
#include <corosync/list.h>
#include <qb/qbdefs.h>
#include <qb/qbloop.h>
#define LOGSYS_UTILS_ONLY 1
#include <corosync/logsys.h>
#include "totemudp.h"
#include "util.h"
#include "totemcrypto.h"
#include <nss.h>
#include <pk11pub.h>
#include <pkcs11.h>
#include <prerror.h>
#ifndef MSG_NOSIGNAL
#define MSG_NOSIGNAL 0
#endif
#define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
#define NETIF_STATE_REPORT_UP 1
#define NETIF_STATE_REPORT_DOWN 2
#define BIND_STATE_UNBOUND 0
#define BIND_STATE_REGULAR 1
#define BIND_STATE_LOOPBACK 2
struct totemudp_socket {
int mcast_recv;
int mcast_send;
int token;
/*
* Socket used for local multicast delivery. We don't rely on multicast
* loop and rather this UNIX DGRAM socket is used. Socket is created by
* socketpair call and they are used in same way as pipe (so [0] is read
* end and [1] is write end)
*/
int local_mcast_loop[2];
};
struct totemudp_instance {
struct crypto_instance *crypto_inst;
qb_loop_t *totemudp_poll_handle;
struct totem_interface *totem_interface;
int netif_state_report;
int netif_bind_state;
void *context;
void (*totemudp_deliver_fn) (
void *context,
const void *msg,
unsigned int msg_len);
void (*totemudp_iface_change_fn) (
void *context,
const struct totem_ip_address *iface_address);
void (*totemudp_target_set_completed) (void *context);
/*
* Function and data used to log messages
*/
int totemudp_log_level_security;
int totemudp_log_level_error;
int totemudp_log_level_warning;
int totemudp_log_level_notice;
int totemudp_log_level_debug;
int totemudp_subsys_id;
void (*totemudp_log_printf) (
int level,
int subsys,
const char *function,
const char *file,
int line,
const char *format,
...)__attribute__((format(printf, 6, 7)));
void *udp_context;
char iov_buffer[FRAME_SIZE_MAX];
char iov_buffer_flush[FRAME_SIZE_MAX];
struct iovec totemudp_iov_recv;
struct iovec totemudp_iov_recv_flush;
struct totemudp_socket totemudp_sockets;
struct totem_ip_address mcast_address;
int stats_sent;
int stats_recv;
int stats_delv;
int stats_remcasts;
int stats_orf_token;
struct timeval stats_tv_start;
struct totem_ip_address my_id;
int firstrun;
qb_loop_timer_handle timer_netif_check_timeout;
unsigned int my_memb_entries;
int flushing;
struct totem_config *totem_config;
totemsrp_stats_t *stats;
struct totem_ip_address token_target;
};
struct work_item {
const void *msg;
unsigned int msg_len;
struct totemudp_instance *instance;
};
static int totemudp_build_sockets (
struct totemudp_instance *instance,
struct totem_ip_address *bindnet_address,
struct totem_ip_address *mcastaddress,
struct totemudp_socket *sockets,
struct totem_ip_address *bound_to);
static struct totem_ip_address localhost;
static void totemudp_instance_initialize (struct totemudp_instance *instance)
{
memset (instance, 0, sizeof (struct totemudp_instance));
instance->netif_state_report = NETIF_STATE_REPORT_UP | NETIF_STATE_REPORT_DOWN;
instance->totemudp_iov_recv.iov_base = instance->iov_buffer;
instance->totemudp_iov_recv.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
instance->totemudp_iov_recv_flush.iov_base = instance->iov_buffer_flush;
instance->totemudp_iov_recv_flush.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
/*
* There is always atleast 1 processor
*/
instance->my_memb_entries = 1;
}
#define log_printf(level, format, args...) \
do { \
instance->totemudp_log_printf ( \
level, instance->totemudp_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
(const char *)format, ##args); \
} while (0);
#define LOGSYS_PERROR(err_num, level, fmt, args...) \
do { \
char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
instance->totemudp_log_printf ( \
level, instance->totemudp_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
fmt ": %s (%d)\n", ##args, _error_ptr, err_num); \
} while(0)
int totemudp_crypto_set (
void *udp_context,
const char *cipher_type,
const char *hash_type)
{
return (0);
}
static inline void ucast_sendmsg (
struct totemudp_instance *instance,
struct totem_ip_address *system_to,
const void *msg,
unsigned int msg_len)
{
struct msghdr msg_ucast;
int res = 0;
size_t buf_out_len;
unsigned char buf_out[FRAME_SIZE_MAX];
struct sockaddr_storage sockaddr;
struct iovec iovec;
int addrlen;
+ if (msg_len + crypto_get_current_sec_header_size(instance->crypto_inst) > sizeof(buf_out)) {
+ log_printf(LOGSYS_LEVEL_CRIT, "UDP message for ucast is too big. Ignoring message");
+
+ return ;
+ }
+
/*
* Encrypt and digest the message
*/
if (crypto_encrypt_and_sign (
instance->crypto_inst,
(const unsigned char *)msg,
msg_len,
buf_out,
&buf_out_len) != 0) {
log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
return;
}
iovec.iov_base = (void *)buf_out;
iovec.iov_len = buf_out_len;
/*
* Build unicast message
*/
memset(&msg_ucast, 0, sizeof(msg_ucast));
totemip_totemip_to_sockaddr_convert(system_to,
instance->totem_interface->ip_port, &sockaddr, &addrlen);
msg_ucast.msg_name = &sockaddr;
msg_ucast.msg_namelen = addrlen;
msg_ucast.msg_iov = (void *)&iovec;
msg_ucast.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_ucast.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_ucast.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_ucast.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_ucast.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_ucast.msg_accrightslen = 0;
#endif
/*
* Transmit unicast message
* An error here is recovered by totemsrp
*/
res = sendmsg (instance->totemudp_sockets.mcast_send, &msg_ucast,
MSG_NOSIGNAL);
if (res < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"sendmsg(ucast) failed (non-critical)");
}
}
static inline void mcast_sendmsg (
struct totemudp_instance *instance,
const void *msg,
unsigned int msg_len)
{
struct msghdr msg_mcast;
int res = 0;
size_t buf_out_len;
unsigned char buf_out[FRAME_SIZE_MAX];
struct iovec iovec;
struct sockaddr_storage sockaddr;
int addrlen;
+ if (msg_len + crypto_get_current_sec_header_size(instance->crypto_inst) > sizeof(buf_out)) {
+ log_printf(LOGSYS_LEVEL_CRIT, "UDP message for mcast is too big. Ignoring message");
+
+ return ;
+ }
+
/*
* Encrypt and digest the message
*/
if (crypto_encrypt_and_sign (
instance->crypto_inst,
(const unsigned char *)msg,
msg_len,
buf_out,
&buf_out_len) != 0) {
log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
return;
}
iovec.iov_base = (void *)&buf_out;
iovec.iov_len = buf_out_len;
/*
* Build multicast message
*/
totemip_totemip_to_sockaddr_convert(&instance->mcast_address,
instance->totem_interface->ip_port, &sockaddr, &addrlen);
memset(&msg_mcast, 0, sizeof(msg_mcast));
msg_mcast.msg_name = &sockaddr;
msg_mcast.msg_namelen = addrlen;
msg_mcast.msg_iov = (void *)&iovec;
msg_mcast.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_mcast.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_mcast.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_mcast.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_mcast.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_mcast.msg_accrightslen = 0;
#endif
/*
* Transmit multicast message
* An error here is recovered by totemsrp
*/
res = sendmsg (instance->totemudp_sockets.mcast_send, &msg_mcast,
MSG_NOSIGNAL);
if (res < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"sendmsg(mcast) failed (non-critical)");
instance->stats->continuous_sendmsg_failures++;
} else {
instance->stats->continuous_sendmsg_failures = 0;
}
/*
* Transmit multicast message to local unix mcast loop
* An error here is recovered by totemsrp
*/
msg_mcast.msg_name = NULL;
msg_mcast.msg_namelen = 0;
res = sendmsg (instance->totemudp_sockets.local_mcast_loop[1], &msg_mcast,
MSG_NOSIGNAL);
if (res < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"sendmsg(local mcast loop) failed (non-critical)");
}
}
int totemudp_finalize (
void *udp_context)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
if (instance->totemudp_sockets.mcast_recv > 0) {
qb_loop_poll_del (instance->totemudp_poll_handle,
instance->totemudp_sockets.mcast_recv);
close (instance->totemudp_sockets.mcast_recv);
}
if (instance->totemudp_sockets.mcast_send > 0) {
close (instance->totemudp_sockets.mcast_send);
}
if (instance->totemudp_sockets.local_mcast_loop[0] > 0) {
qb_loop_poll_del (instance->totemudp_poll_handle,
instance->totemudp_sockets.local_mcast_loop[0]);
close (instance->totemudp_sockets.local_mcast_loop[0]);
close (instance->totemudp_sockets.local_mcast_loop[1]);
}
if (instance->totemudp_sockets.token > 0) {
qb_loop_poll_del (instance->totemudp_poll_handle,
instance->totemudp_sockets.token);
close (instance->totemudp_sockets.token);
}
return (res);
}
/*
* Only designed to work with a message with one iov
*/
static int net_deliver_fn (
int fd,
int revents,
void *data)
{
struct totemudp_instance *instance = (struct totemudp_instance *)data;
struct msghdr msg_recv;
struct iovec *iovec;
struct sockaddr_storage system_from;
int bytes_received;
int res = 0;
int truncated_packet;
if (instance->flushing == 1) {
iovec = &instance->totemudp_iov_recv_flush;
} else {
iovec = &instance->totemudp_iov_recv;
}
/*
* Receive datagram
*/
msg_recv.msg_name = &system_from;
msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
msg_recv.msg_iov = iovec;
msg_recv.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_recv.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_recv.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_recv.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_recv.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_recv.msg_accrightslen = 0;
#endif
bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
if (bytes_received == -1) {
return (0);
} else {
instance->stats_recv += bytes_received;
}
truncated_packet = 0;
#ifdef HAVE_MSGHDR_FLAGS
if (msg_recv.msg_flags & MSG_TRUNC) {
truncated_packet = 1;
}
#else
/*
* We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
* if bytes_received == FRAME_SIZE_MAX then packet is truncated
*/
if (bytes_received == FRAME_SIZE_MAX) {
truncated_packet = 1;
}
#endif
if (truncated_packet) {
log_printf(instance->totemudp_log_level_error,
"Received too big message. This may be because something bad is happening"
"on the network (attack?), or you tried join more nodes than corosync is"
"compiled with (%u) or bug in the code (bad estimation of "
"the FRAME_SIZE_MAX). Dropping packet.", PROCESSOR_COUNT_MAX);
return (0);
}
/*
* Authenticate and if authenticated, decrypt datagram
*/
res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
if (res == -1) {
log_printf (instance->totemudp_log_level_security, "Received message has invalid digest... ignoring.");
log_printf (instance->totemudp_log_level_security,
"Invalid packet data");
iovec->iov_len = FRAME_SIZE_MAX;
return 0;
}
iovec->iov_len = bytes_received;
/*
* Handle incoming message
*/
instance->totemudp_deliver_fn (
instance->context,
iovec->iov_base,
iovec->iov_len);
iovec->iov_len = FRAME_SIZE_MAX;
return (0);
}
static int netif_determine (
struct totemudp_instance *instance,
struct totem_ip_address *bindnet,
struct totem_ip_address *bound_to,
int *interface_up,
int *interface_num)
{
int res;
res = totemip_iface_check (bindnet, bound_to,
interface_up, interface_num,
instance->totem_config->clear_node_high_bit);
return (res);
}
/*
* If the interface is up, the sockets for totem are built. If the interface is down
* this function is requeued in the timer list to retry building the sockets later.
*/
static void timer_function_netif_check_timeout (
void *data)
{
struct totemudp_instance *instance = (struct totemudp_instance *)data;
int interface_up;
int interface_num;
struct totem_ip_address *bind_address;
/*
* Build sockets for every interface
*/
netif_determine (instance,
&instance->totem_interface->bindnet,
&instance->totem_interface->boundto,
&interface_up, &interface_num);
/*
* If the network interface isn't back up and we are already
* in loopback mode, add timer to check again and return
*/
if ((instance->netif_bind_state == BIND_STATE_LOOPBACK &&
interface_up == 0) ||
(instance->my_memb_entries == 1 &&
instance->netif_bind_state == BIND_STATE_REGULAR &&
interface_up == 1)) {
qb_loop_timer_add (instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
/*
* Add a timer to check for a downed regular interface
*/
return;
}
if (instance->totemudp_sockets.mcast_recv > 0) {
qb_loop_poll_del (instance->totemudp_poll_handle,
instance->totemudp_sockets.mcast_recv);
close (instance->totemudp_sockets.mcast_recv);
}
if (instance->totemudp_sockets.mcast_send > 0) {
close (instance->totemudp_sockets.mcast_send);
}
if (instance->totemudp_sockets.local_mcast_loop[0] > 0) {
qb_loop_poll_del (instance->totemudp_poll_handle,
instance->totemudp_sockets.local_mcast_loop[0]);
close (instance->totemudp_sockets.local_mcast_loop[0]);
close (instance->totemudp_sockets.local_mcast_loop[1]);
}
if (instance->totemudp_sockets.token > 0) {
qb_loop_poll_del (instance->totemudp_poll_handle,
instance->totemudp_sockets.token);
close (instance->totemudp_sockets.token);
}
if (interface_up == 0) {
/*
* Interface is not up
*/
instance->netif_bind_state = BIND_STATE_LOOPBACK;
bind_address = &localhost;
/*
* Add a timer to retry building interfaces and request memb_gather_enter
*/
qb_loop_timer_add (instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
} else {
/*
* Interface is up
*/
instance->netif_bind_state = BIND_STATE_REGULAR;
bind_address = &instance->totem_interface->bindnet;
}
/*
* Create and bind the multicast and unicast sockets
*/
(void)totemudp_build_sockets (instance,
&instance->mcast_address,
bind_address,
&instance->totemudp_sockets,
&instance->totem_interface->boundto);
qb_loop_poll_add (
instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totemudp_sockets.mcast_recv,
POLLIN, instance, net_deliver_fn);
qb_loop_poll_add (
instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totemudp_sockets.local_mcast_loop[0],
POLLIN, instance, net_deliver_fn);
qb_loop_poll_add (
instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totemudp_sockets.token,
POLLIN, instance, net_deliver_fn);
totemip_copy (&instance->my_id, &instance->totem_interface->boundto);
/*
* This reports changes in the interface to the user and totemsrp
*/
if (instance->netif_bind_state == BIND_STATE_REGULAR) {
if (instance->netif_state_report & NETIF_STATE_REPORT_UP) {
log_printf (instance->totemudp_log_level_notice,
"The network interface [%s] is now up.",
totemip_print (&instance->totem_interface->boundto));
instance->netif_state_report = NETIF_STATE_REPORT_DOWN;
instance->totemudp_iface_change_fn (instance->context, &instance->my_id);
}
/*
* Add a timer to check for interface going down in single membership
*/
if (instance->my_memb_entries == 1) {
qb_loop_timer_add (instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
}
} else {
if (instance->netif_state_report & NETIF_STATE_REPORT_DOWN) {
log_printf (instance->totemudp_log_level_notice,
"The network interface is down.");
instance->totemudp_iface_change_fn (instance->context, &instance->my_id);
}
instance->netif_state_report = NETIF_STATE_REPORT_UP;
}
}
/* Set the socket priority to INTERACTIVE to ensure
that our messages don't get queued behind anything else */
static void totemudp_traffic_control_set(struct totemudp_instance *instance, int sock)
{
#ifdef SO_PRIORITY
int prio = 6; /* TC_PRIO_INTERACTIVE */
if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(int))) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning, "Could not set traffic priority");
}
#endif
}
static int totemudp_build_sockets_ip (
struct totemudp_instance *instance,
struct totem_ip_address *mcast_address,
struct totem_ip_address *bindnet_address,
struct totemudp_socket *sockets,
struct totem_ip_address *bound_to,
int interface_num)
{
struct sockaddr_storage sockaddr;
struct ipv6_mreq mreq6;
struct ip_mreq mreq;
struct sockaddr_storage mcast_ss, boundto_ss;
struct sockaddr_in6 *mcast_sin6 = (struct sockaddr_in6 *)&mcast_ss;
struct sockaddr_in *mcast_sin = (struct sockaddr_in *)&mcast_ss;
struct sockaddr_in *boundto_sin = (struct sockaddr_in *)&boundto_ss;
unsigned int sendbuf_size;
unsigned int recvbuf_size;
unsigned int optlen = sizeof (sendbuf_size);
unsigned int retries;
int addrlen;
int res;
int flag;
uint8_t sflag;
int i;
/*
* Create multicast recv socket
*/
sockets->mcast_recv = socket (bindnet_address->family, SOCK_DGRAM, 0);
if (sockets->mcast_recv == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"socket() failed");
return (-1);
}
totemip_nosigpipe (sockets->mcast_recv);
res = fcntl (sockets->mcast_recv, F_SETFL, O_NONBLOCK);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Could not set non-blocking operation on multicast socket");
return (-1);
}
/*
* Force reuse
*/
flag = 1;
if ( setsockopt(sockets->mcast_recv, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"setsockopt(SO_REUSEADDR) failed");
return (-1);
}
/*
* Create local multicast loop socket
*/
if (socketpair(AF_UNIX, SOCK_DGRAM, 0, sockets->local_mcast_loop) == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"socket() failed");
return (-1);
}
for (i = 0; i < 2; i++) {
totemip_nosigpipe (sockets->local_mcast_loop[i]);
res = fcntl (sockets->local_mcast_loop[i], F_SETFL, O_NONBLOCK);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Could not set non-blocking operation on multicast socket");
return (-1);
}
}
/*
* Setup mcast send socket
*/
sockets->mcast_send = socket (bindnet_address->family, SOCK_DGRAM, 0);
if (sockets->mcast_send == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"socket() failed");
return (-1);
}
totemip_nosigpipe (sockets->mcast_send);
res = fcntl (sockets->mcast_send, F_SETFL, O_NONBLOCK);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Could not set non-blocking operation on multicast socket");
return (-1);
}
/*
* Force reuse
*/
flag = 1;
if ( setsockopt(sockets->mcast_send, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"setsockopt(SO_REUSEADDR) failed");
return (-1);
}
totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port - 1,
&sockaddr, &addrlen);
retries = 0;
while (1) {
res = bind (sockets->mcast_send, (struct sockaddr *)&sockaddr, addrlen);
if (res == 0) {
break;
}
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Unable to bind the socket to send multicast packets");
if (++retries > BIND_MAX_RETRIES) {
break;
}
/*
* Wait for a while
*/
(void)poll(NULL, 0, BIND_RETRIES_INTERVAL * retries);
}
if (res == -1) {
return (-1);
}
/*
* Setup unicast socket
*/
sockets->token = socket (bindnet_address->family, SOCK_DGRAM, 0);
if (sockets->token == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"socket() failed");
return (-1);
}
totemip_nosigpipe (sockets->token);
res = fcntl (sockets->token, F_SETFL, O_NONBLOCK);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Could not set non-blocking operation on token socket");
return (-1);
}
/*
* Force reuse
*/
flag = 1;
if ( setsockopt(sockets->token, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"setsockopt(SO_REUSEADDR) failed");
return (-1);
}
/*
* Bind to unicast socket used for token send/receives
* This has the side effect of binding to the correct interface
*/
totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &sockaddr, &addrlen);
retries = 0;
while (1) {
res = bind (sockets->token, (struct sockaddr *)&sockaddr, addrlen);
if (res == 0) {
break;
}
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Unable to bind UDP unicast socket");
if (++retries > BIND_MAX_RETRIES) {
break;
}
/*
* Wait for a while
*/
(void)poll(NULL, 0, BIND_RETRIES_INTERVAL * retries);
}
if (res == -1) {
return (-1);
}
recvbuf_size = MCAST_SOCKET_BUFFER_SIZE;
sendbuf_size = MCAST_SOCKET_BUFFER_SIZE;
/*
* Set buffer sizes to avoid overruns
*/
res = setsockopt (sockets->mcast_recv, SOL_SOCKET, SO_RCVBUF, &recvbuf_size, optlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"Unable to set SO_RCVBUF size on UDP mcast socket");
return (-1);
}
res = setsockopt (sockets->mcast_send, SOL_SOCKET, SO_SNDBUF, &sendbuf_size, optlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"Unable to set SO_SNDBUF size on UDP mcast socket");
return (-1);
}
res = setsockopt (sockets->local_mcast_loop[0], SOL_SOCKET, SO_RCVBUF, &recvbuf_size, optlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"Unable to set SO_RCVBUF size on UDP local mcast loop socket");
return (-1);
}
res = setsockopt (sockets->local_mcast_loop[1], SOL_SOCKET, SO_SNDBUF, &sendbuf_size, optlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_debug,
"Unable to set SO_SNDBUF size on UDP local mcast loop socket");
return (-1);
}
res = getsockopt (sockets->mcast_recv, SOL_SOCKET, SO_RCVBUF, &recvbuf_size, &optlen);
if (res == 0) {
log_printf (instance->totemudp_log_level_debug,
"Receive multicast socket recv buffer size (%d bytes).", recvbuf_size);
}
res = getsockopt (sockets->mcast_send, SOL_SOCKET, SO_SNDBUF, &sendbuf_size, &optlen);
if (res == 0) {
log_printf (instance->totemudp_log_level_debug,
"Transmit multicast socket send buffer size (%d bytes).", sendbuf_size);
}
res = getsockopt (sockets->local_mcast_loop[0], SOL_SOCKET, SO_RCVBUF, &recvbuf_size, &optlen);
if (res == 0) {
log_printf (instance->totemudp_log_level_debug,
"Local receive multicast loop socket recv buffer size (%d bytes).", recvbuf_size);
}
res = getsockopt (sockets->local_mcast_loop[1], SOL_SOCKET, SO_SNDBUF, &sendbuf_size, &optlen);
if (res == 0) {
log_printf (instance->totemudp_log_level_debug,
"Local transmit multicast loop socket send buffer size (%d bytes).", sendbuf_size);
}
/*
* Join group membership on socket
*/
totemip_totemip_to_sockaddr_convert(mcast_address, instance->totem_interface->ip_port, &mcast_ss, &addrlen);
totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &boundto_ss, &addrlen);
if (instance->totem_config->broadcast_use == 1) {
unsigned int broadcast = 1;
if ((setsockopt(sockets->mcast_recv, SOL_SOCKET,
SO_BROADCAST, &broadcast, sizeof (broadcast))) == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"setting broadcast option failed");
return (-1);
}
if ((setsockopt(sockets->mcast_send, SOL_SOCKET,
SO_BROADCAST, &broadcast, sizeof (broadcast))) == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"setting broadcast option failed");
return (-1);
}
} else {
switch (bindnet_address->family) {
case AF_INET:
memset(&mreq, 0, sizeof(mreq));
mreq.imr_multiaddr.s_addr = mcast_sin->sin_addr.s_addr;
mreq.imr_interface.s_addr = boundto_sin->sin_addr.s_addr;
res = setsockopt (sockets->mcast_recv, IPPROTO_IP, IP_ADD_MEMBERSHIP,
&mreq, sizeof (mreq));
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"join ipv4 multicast group failed");
return (-1);
}
break;
case AF_INET6:
memset(&mreq6, 0, sizeof(mreq6));
memcpy(&mreq6.ipv6mr_multiaddr, &mcast_sin6->sin6_addr, sizeof(struct in6_addr));
mreq6.ipv6mr_interface = interface_num;
res = setsockopt (sockets->mcast_recv, IPPROTO_IPV6, IPV6_JOIN_GROUP,
&mreq6, sizeof (mreq6));
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"join ipv6 multicast group failed");
return (-1);
}
break;
}
}
/*
* Turn off multicast loopback
*/
flag = 0;
switch ( bindnet_address->family ) {
case AF_INET:
sflag = 0;
res = setsockopt (sockets->mcast_send, IPPROTO_IP, IP_MULTICAST_LOOP,
&sflag, sizeof (sflag));
break;
case AF_INET6:
res = setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_LOOP,
&flag, sizeof (flag));
}
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Unable to turn off multicast loopback");
return (-1);
}
/*
* Set multicast packets TTL
*/
flag = instance->totem_interface->ttl;
if (bindnet_address->family == AF_INET6) {
res = setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_HOPS,
&flag, sizeof (flag));
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"set mcast v6 TTL failed");
return (-1);
}
} else {
sflag = flag;
res = setsockopt(sockets->mcast_send, IPPROTO_IP, IP_MULTICAST_TTL,
&sflag, sizeof(sflag));
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"set mcast v4 TTL failed");
return (-1);
}
}
/*
* Bind to a specific interface for multicast send and receive
*/
switch ( bindnet_address->family ) {
case AF_INET:
if (setsockopt (sockets->mcast_send, IPPROTO_IP, IP_MULTICAST_IF,
&boundto_sin->sin_addr, sizeof (boundto_sin->sin_addr)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"cannot select interface for multicast packets (send)");
return (-1);
}
if (setsockopt (sockets->mcast_recv, IPPROTO_IP, IP_MULTICAST_IF,
&boundto_sin->sin_addr, sizeof (boundto_sin->sin_addr)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"cannot select interface for multicast packets (recv)");
return (-1);
}
break;
case AF_INET6:
if (setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_IF,
&interface_num, sizeof (interface_num)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"cannot select interface for multicast packets (send v6)");
return (-1);
}
if (setsockopt (sockets->mcast_recv, IPPROTO_IPV6, IPV6_MULTICAST_IF,
&interface_num, sizeof (interface_num)) < 0) {
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"cannot select interface for multicast packets (recv v6)");
return (-1);
}
break;
}
/*
* Bind to multicast socket used for multicast receives
* This needs to happen after all of the multicast setsockopt() calls
* as the kernel seems to only put them into effect (for IPV6) when bind()
* is called.
*/
totemip_totemip_to_sockaddr_convert(mcast_address,
instance->totem_interface->ip_port, &sockaddr, &addrlen);
retries = 0;
while (1) {
res = bind (sockets->mcast_recv, (struct sockaddr *)&sockaddr, addrlen);
if (res == 0) {
break;
}
LOGSYS_PERROR (errno, instance->totemudp_log_level_warning,
"Unable to bind the socket to receive multicast packets");
if (++retries > BIND_MAX_RETRIES) {
break;
}
/*
* Wait for a while
*/
(void)poll(NULL, 0, BIND_RETRIES_INTERVAL * retries);
}
if (res == -1) {
return (-1);
}
return 0;
}
static int totemudp_build_sockets (
struct totemudp_instance *instance,
struct totem_ip_address *mcast_address,
struct totem_ip_address *bindnet_address,
struct totemudp_socket *sockets,
struct totem_ip_address *bound_to)
{
int interface_num;
int interface_up;
int res;
/*
* Determine the ip address bound to and the interface name
*/
res = netif_determine (instance,
bindnet_address,
bound_to,
&interface_up,
&interface_num);
if (res == -1) {
return (-1);
}
totemip_copy(&instance->my_id, bound_to);
res = totemudp_build_sockets_ip (instance, mcast_address,
bindnet_address, sockets, bound_to, interface_num);
if (res == -1) {
/* if we get here, corosync won't work anyway, so better leaving than faking to work */
LOGSYS_PERROR (errno, instance->totemudp_log_level_error,
"Unable to create sockets, exiting");
exit(EXIT_FAILURE);
}
/* We only send out of the token socket */
totemudp_traffic_control_set(instance, sockets->token);
return res;
}
/*
* Totem Network interface - also does encryption/decryption
* depends on poll abstraction, POSIX, IPV4
*/
/*
* Create an instance
*/
int totemudp_initialize (
qb_loop_t *poll_handle,
void **udp_context,
struct totem_config *totem_config,
totemsrp_stats_t *stats,
int interface_no,
void *context,
void (*deliver_fn) (
void *context,
const void *msg,
unsigned int msg_len),
void (*iface_change_fn) (
void *context,
const struct totem_ip_address *iface_address),
void (*target_set_completed) (
void *context))
{
struct totemudp_instance *instance;
instance = malloc (sizeof (struct totemudp_instance));
if (instance == NULL) {
return (-1);
}
totemudp_instance_initialize (instance);
instance->totem_config = totem_config;
instance->stats = stats;
/*
* Configure logging
*/
instance->totemudp_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
instance->totemudp_log_level_error = totem_config->totem_logging_configuration.log_level_error;
instance->totemudp_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
instance->totemudp_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
instance->totemudp_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
instance->totemudp_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
instance->totemudp_log_printf = totem_config->totem_logging_configuration.log_printf;
/*
* Initialize random number generator for later use to generate salt
*/
instance->crypto_inst = crypto_init (totem_config->private_key,
totem_config->private_key_len,
totem_config->crypto_cipher_type,
totem_config->crypto_hash_type,
instance->totemudp_log_printf,
instance->totemudp_log_level_security,
instance->totemudp_log_level_notice,
instance->totemudp_log_level_error,
instance->totemudp_subsys_id);
if (instance->crypto_inst == NULL) {
free(instance);
return (-1);
}
/*
* Initialize local variables for totemudp
*/
instance->totem_interface = &totem_config->interfaces[interface_no];
totemip_copy (&instance->mcast_address, &instance->totem_interface->mcast_addr);
memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
instance->totemudp_poll_handle = poll_handle;
instance->totem_interface->bindnet.nodeid = instance->totem_config->node_id;
instance->context = context;
instance->totemudp_deliver_fn = deliver_fn;
instance->totemudp_iface_change_fn = iface_change_fn;
instance->totemudp_target_set_completed = target_set_completed;
totemip_localhost (instance->mcast_address.family, &localhost);
localhost.nodeid = instance->totem_config->node_id;
/*
* RRP layer isn't ready to receive message because it hasn't
* initialized yet. Add short timer to check the interfaces.
*/
qb_loop_timer_add (instance->totemudp_poll_handle,
QB_LOOP_MED,
100*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
*udp_context = instance;
return (0);
}
void *totemudp_buffer_alloc (void)
{
return malloc (FRAME_SIZE_MAX);
}
void totemudp_buffer_release (void *ptr)
{
return free (ptr);
}
int totemudp_processor_count_set (
void *udp_context,
int processor_count)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
instance->my_memb_entries = processor_count;
qb_loop_timer_del (instance->totemudp_poll_handle,
instance->timer_netif_check_timeout);
if (processor_count == 1) {
qb_loop_timer_add (instance->totemudp_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
}
return (res);
}
int totemudp_recv_flush (void *udp_context)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
struct pollfd ufd;
int nfds;
int res = 0;
int i;
int sock;
instance->flushing = 1;
for (i = 0; i < 2; i++) {
sock = -1;
if (i == 0) {
sock = instance->totemudp_sockets.mcast_recv;
}
if (i == 1) {
sock = instance->totemudp_sockets.local_mcast_loop[0];
}
assert(sock != -1);
do {
ufd.fd = sock;
ufd.events = POLLIN;
nfds = poll (&ufd, 1, 0);
if (nfds == 1 && ufd.revents & POLLIN) {
net_deliver_fn (sock, ufd.revents, instance);
}
} while (nfds == 1);
}
instance->flushing = 0;
return (res);
}
int totemudp_send_flush (void *udp_context)
{
return 0;
}
int totemudp_token_send (
void *udp_context,
const void *msg,
unsigned int msg_len)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
return (res);
}
int totemudp_mcast_flush_send (
void *udp_context,
const void *msg,
unsigned int msg_len)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
mcast_sendmsg (instance, msg, msg_len);
return (res);
}
int totemudp_mcast_noflush_send (
void *udp_context,
const void *msg,
unsigned int msg_len)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
mcast_sendmsg (instance, msg, msg_len);
return (res);
}
extern int totemudp_iface_check (void *udp_context)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
timer_function_netif_check_timeout (instance);
return (res);
}
extern void totemudp_net_mtu_adjust (void *udp_context, struct totem_config *totem_config)
{
assert(totem_config->interface_count > 0);
totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
totem_config->crypto_hash_type) +
totemip_udpip_header_size(totem_config->interfaces[0].bindnet.family);
}
const char *totemudp_iface_print (void *udp_context) {
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
const char *ret_char;
ret_char = totemip_print (&instance->my_id);
return (ret_char);
}
int totemudp_iface_get (
void *udp_context,
struct totem_ip_address *addr)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
return (res);
}
int totemudp_token_target_set (
void *udp_context,
const struct totem_ip_address *token_target)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
int res = 0;
memcpy (&instance->token_target, token_target,
sizeof (struct totem_ip_address));
instance->totemudp_target_set_completed (instance->context);
return (res);
}
extern int totemudp_recv_mcast_empty (
void *udp_context)
{
struct totemudp_instance *instance = (struct totemudp_instance *)udp_context;
unsigned int res;
struct sockaddr_storage system_from;
struct msghdr msg_recv;
struct pollfd ufd;
int nfds;
int msg_processed = 0;
int i;
int sock;
/*
* Receive datagram
*/
msg_recv.msg_name = &system_from;
msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
msg_recv.msg_iov = &instance->totemudp_iov_recv_flush;
msg_recv.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_recv.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_recv.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_recv.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_recv.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_recv.msg_accrightslen = 0;
#endif
for (i = 0; i < 2; i++) {
sock = -1;
if (i == 0) {
sock = instance->totemudp_sockets.mcast_recv;
}
if (i == 1) {
sock = instance->totemudp_sockets.local_mcast_loop[0];
}
assert(sock != -1);
do {
ufd.fd = sock;
ufd.events = POLLIN;
nfds = poll (&ufd, 1, 0);
if (nfds == 1 && ufd.revents & POLLIN) {
res = recvmsg (sock, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
if (res != -1) {
msg_processed = 1;
} else {
msg_processed = -1;
}
}
} while (nfds == 1);
}
return (msg_processed);
}
diff --git a/exec/totemudpu.c b/exec/totemudpu.c
index 569e67a0..a5a81648 100644
--- a/exec/totemudpu.c
+++ b/exec/totemudpu.c
@@ -1,1334 +1,1346 @@
/*
* Copyright (c) 2005 MontaVista Software, Inc.
* Copyright (c) 2006-2012 Red Hat, Inc.
*
* All rights reserved.
*
* Author: Steven Dake (sdake@redhat.com)
* This software licensed under BSD license, the text of which follows:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* - Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* - Neither the name of the MontaVista Software, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <config.h>
#include <assert.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <netdb.h>
#include <sys/un.h>
#include <sys/ioctl.h>
#include <sys/param.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <sched.h>
#include <time.h>
#include <sys/time.h>
#include <sys/poll.h>
#include <sys/uio.h>
#include <limits.h>
#include <qb/qbdefs.h>
#include <qb/qbloop.h>
#include <corosync/sq.h>
#include <corosync/list.h>
#include <corosync/swab.h>
#define LOGSYS_UTILS_ONLY 1
#include <corosync/logsys.h>
#include "totemudpu.h"
#include "util.h"
#include "totemcrypto.h"
#include <nss.h>
#include <pk11pub.h>
#include <pkcs11.h>
#include <prerror.h>
#ifndef MSG_NOSIGNAL
#define MSG_NOSIGNAL 0
#endif
#define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
#define NETIF_STATE_REPORT_UP 1
#define NETIF_STATE_REPORT_DOWN 2
#define BIND_STATE_UNBOUND 0
#define BIND_STATE_REGULAR 1
#define BIND_STATE_LOOPBACK 2
struct totemudpu_member {
struct list_head list;
struct totem_ip_address member;
int fd;
int active;
};
struct totemudpu_instance {
struct crypto_instance *crypto_inst;
qb_loop_t *totemudpu_poll_handle;
struct totem_interface *totem_interface;
int netif_state_report;
int netif_bind_state;
void *context;
void (*totemudpu_deliver_fn) (
void *context,
const void *msg,
unsigned int msg_len);
void (*totemudpu_iface_change_fn) (
void *context,
const struct totem_ip_address *iface_address);
void (*totemudpu_target_set_completed) (void *context);
/*
* Function and data used to log messages
*/
int totemudpu_log_level_security;
int totemudpu_log_level_error;
int totemudpu_log_level_warning;
int totemudpu_log_level_notice;
int totemudpu_log_level_debug;
int totemudpu_subsys_id;
void (*totemudpu_log_printf) (
int level,
int subsys,
const char *function,
const char *file,
int line,
const char *format,
...)__attribute__((format(printf, 6, 7)));
void *udpu_context;
char iov_buffer[FRAME_SIZE_MAX];
struct iovec totemudpu_iov_recv;
struct list_head member_list;
int stats_sent;
int stats_recv;
int stats_delv;
int stats_remcasts;
int stats_orf_token;
struct timeval stats_tv_start;
struct totem_ip_address my_id;
int firstrun;
qb_loop_timer_handle timer_netif_check_timeout;
unsigned int my_memb_entries;
struct totem_config *totem_config;
totemsrp_stats_t *stats;
struct totem_ip_address token_target;
int token_socket;
qb_loop_timer_handle timer_merge_detect_timeout;
int send_merge_detect_message;
unsigned int merge_detect_messages_sent_before_timeout;
};
struct work_item {
const void *msg;
unsigned int msg_len;
struct totemudpu_instance *instance;
};
static int totemudpu_build_sockets (
struct totemudpu_instance *instance,
struct totem_ip_address *bindnet_address,
struct totem_ip_address *bound_to);
static int totemudpu_create_sending_socket(
void *udpu_context,
const struct totem_ip_address *member);
int totemudpu_member_list_rebind_ip (
void *udpu_context);
static void totemudpu_start_merge_detect_timeout(
void *udpu_context);
static void totemudpu_stop_merge_detect_timeout(
void *udpu_context);
static struct totem_ip_address localhost;
static void totemudpu_instance_initialize (struct totemudpu_instance *instance)
{
memset (instance, 0, sizeof (struct totemudpu_instance));
instance->netif_state_report = NETIF_STATE_REPORT_UP | NETIF_STATE_REPORT_DOWN;
instance->totemudpu_iov_recv.iov_base = instance->iov_buffer;
instance->totemudpu_iov_recv.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
/*
* There is always atleast 1 processor
*/
instance->my_memb_entries = 1;
list_init (&instance->member_list);
}
#define log_printf(level, format, args...) \
do { \
instance->totemudpu_log_printf ( \
level, instance->totemudpu_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
(const char *)format, ##args); \
} while (0);
#define LOGSYS_PERROR(err_num, level, fmt, args...) \
do { \
char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
instance->totemudpu_log_printf ( \
level, instance->totemudpu_subsys_id, \
__FUNCTION__, __FILE__, __LINE__, \
fmt ": %s (%d)", ##args, _error_ptr, err_num); \
} while(0)
int totemudpu_crypto_set (
void *udpu_context,
const char *cipher_type,
const char *hash_type)
{
return (0);
}
static inline void ucast_sendmsg (
struct totemudpu_instance *instance,
struct totem_ip_address *system_to,
const void *msg,
unsigned int msg_len)
{
struct msghdr msg_ucast;
int res = 0;
size_t buf_out_len;
unsigned char buf_out[FRAME_SIZE_MAX];
struct sockaddr_storage sockaddr;
struct iovec iovec;
int addrlen;
+ if (msg_len + crypto_get_current_sec_header_size(instance->crypto_inst) > sizeof(buf_out)) {
+ log_printf(LOGSYS_LEVEL_CRIT, "UDPU message for ucast is too big. Ignoring message");
+
+ return ;
+ }
+
/*
* Encrypt and digest the message
*/
if (crypto_encrypt_and_sign (
instance->crypto_inst,
(const unsigned char *)msg,
msg_len,
buf_out,
&buf_out_len) != 0) {
log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
return;
}
iovec.iov_base = (void *)buf_out;
iovec.iov_len = buf_out_len;
/*
* Build unicast message
*/
totemip_totemip_to_sockaddr_convert(system_to,
instance->totem_interface->ip_port, &sockaddr, &addrlen);
memset(&msg_ucast, 0, sizeof(msg_ucast));
msg_ucast.msg_name = &sockaddr;
msg_ucast.msg_namelen = addrlen;
msg_ucast.msg_iov = (void *)&iovec;
msg_ucast.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_ucast.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_ucast.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_ucast.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_ucast.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_ucast.msg_accrightslen = 0;
#endif
/*
* Transmit unicast message
* An error here is recovered by totemsrp
*/
res = sendmsg (instance->token_socket, &msg_ucast, MSG_NOSIGNAL);
if (res < 0) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
"sendmsg(ucast) failed (non-critical)");
}
}
static inline void mcast_sendmsg (
struct totemudpu_instance *instance,
const void *msg,
unsigned int msg_len,
int only_active)
{
struct msghdr msg_mcast;
int res = 0;
size_t buf_out_len;
unsigned char buf_out[FRAME_SIZE_MAX];
struct iovec iovec;
struct sockaddr_storage sockaddr;
int addrlen;
struct list_head *list;
struct totemudpu_member *member;
+ if (msg_len + crypto_get_current_sec_header_size(instance->crypto_inst) > sizeof(buf_out)) {
+ log_printf(LOGSYS_LEVEL_CRIT, "UDPU message for mcast is too big. Ignoring message");
+
+ return ;
+ }
+
/*
* Encrypt and digest the message
*/
if (crypto_encrypt_and_sign (
instance->crypto_inst,
(const unsigned char *)msg,
msg_len,
buf_out,
&buf_out_len) != 0) {
log_printf(LOGSYS_LEVEL_CRIT, "Error encrypting/signing packet (non-critical)");
return;
}
iovec.iov_base = (void *)buf_out;
iovec.iov_len = buf_out_len;
memset(&msg_mcast, 0, sizeof(msg_mcast));
/*
* Build multicast message
*/
for (list = instance->member_list.next;
list != &instance->member_list;
list = list->next) {
member = list_entry (list,
struct totemudpu_member,
list);
/*
* Do not send multicast message if message is not "flush", member
* is inactive and timeout for sending merge message didn't expired.
*/
if (only_active && !member->active && !instance->send_merge_detect_message)
continue ;
totemip_totemip_to_sockaddr_convert(&member->member,
instance->totem_interface->ip_port, &sockaddr, &addrlen);
msg_mcast.msg_name = &sockaddr;
msg_mcast.msg_namelen = addrlen;
msg_mcast.msg_iov = (void *)&iovec;
msg_mcast.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_mcast.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_mcast.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_mcast.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_mcast.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_mcast.msg_accrightslen = 0;
#endif
/*
* Transmit multicast message
* An error here is recovered by totemsrp
*/
res = sendmsg (member->fd, &msg_mcast, MSG_NOSIGNAL);
if (res < 0) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_debug,
"sendmsg(mcast) failed (non-critical)");
}
}
if (!only_active || instance->send_merge_detect_message) {
/*
* Current message was sent to all nodes
*/
instance->merge_detect_messages_sent_before_timeout++;
instance->send_merge_detect_message = 0;
}
}
int totemudpu_finalize (
void *udpu_context)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
if (instance->token_socket > 0) {
qb_loop_poll_del (instance->totemudpu_poll_handle,
instance->token_socket);
close (instance->token_socket);
}
totemudpu_stop_merge_detect_timeout(instance);
return (res);
}
static int net_deliver_fn (
int fd,
int revents,
void *data)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
struct msghdr msg_recv;
struct iovec *iovec;
struct sockaddr_storage system_from;
int bytes_received;
int res = 0;
int truncated_packet;
iovec = &instance->totemudpu_iov_recv;
/*
* Receive datagram
*/
msg_recv.msg_name = &system_from;
msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
msg_recv.msg_iov = iovec;
msg_recv.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_recv.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_recv.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_recv.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_recv.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_recv.msg_accrightslen = 0;
#endif
bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
if (bytes_received == -1) {
return (0);
} else {
instance->stats_recv += bytes_received;
}
truncated_packet = 0;
#ifdef HAVE_MSGHDR_FLAGS
if (msg_recv.msg_flags & MSG_TRUNC) {
truncated_packet = 1;
}
#else
/*
* We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
* if bytes_received == FRAME_SIZE_MAX then packet is truncated
*/
if (bytes_received == FRAME_SIZE_MAX) {
truncated_packet = 1;
}
#endif
if (truncated_packet) {
log_printf(instance->totemudpu_log_level_error,
"Received too big message. This may be because something bad is happening"
"on the network (attack?), or you tried join more nodes than corosync is"
"compiled with (%u) or bug in the code (bad estimation of "
"the FRAME_SIZE_MAX). Dropping packet.", PROCESSOR_COUNT_MAX);
return (0);
}
/*
* Authenticate and if authenticated, decrypt datagram
*/
res = crypto_authenticate_and_decrypt (instance->crypto_inst, iovec->iov_base, &bytes_received);
if (res == -1) {
log_printf (instance->totemudpu_log_level_security, "Received message has invalid digest... ignoring.");
log_printf (instance->totemudpu_log_level_security,
"Invalid packet data");
iovec->iov_len = FRAME_SIZE_MAX;
return 0;
}
iovec->iov_len = bytes_received;
/*
* Handle incoming message
*/
instance->totemudpu_deliver_fn (
instance->context,
iovec->iov_base,
iovec->iov_len);
iovec->iov_len = FRAME_SIZE_MAX;
return (0);
}
static int netif_determine (
struct totemudpu_instance *instance,
struct totem_ip_address *bindnet,
struct totem_ip_address *bound_to,
int *interface_up,
int *interface_num)
{
int res;
res = totemip_iface_check (bindnet, bound_to,
interface_up, interface_num,
instance->totem_config->clear_node_high_bit);
return (res);
}
/*
* If the interface is up, the sockets for totem are built. If the interface is down
* this function is requeued in the timer list to retry building the sockets later.
*/
static void timer_function_netif_check_timeout (
void *data)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
int interface_up;
int interface_num;
struct totem_ip_address *bind_address;
/*
* Build sockets for every interface
*/
netif_determine (instance,
&instance->totem_interface->bindnet,
&instance->totem_interface->boundto,
&interface_up, &interface_num);
/*
* If the network interface isn't back up and we are already
* in loopback mode, add timer to check again and return
*/
if ((instance->netif_bind_state == BIND_STATE_LOOPBACK &&
interface_up == 0) ||
(instance->my_memb_entries == 1 &&
instance->netif_bind_state == BIND_STATE_REGULAR &&
interface_up == 1)) {
qb_loop_timer_add (instance->totemudpu_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
/*
* Add a timer to check for a downed regular interface
*/
return;
}
if (instance->token_socket > 0) {
qb_loop_poll_del (instance->totemudpu_poll_handle,
instance->token_socket);
close (instance->token_socket);
}
if (interface_up == 0) {
/*
* Interface is not up
*/
instance->netif_bind_state = BIND_STATE_LOOPBACK;
bind_address = &localhost;
/*
* Add a timer to retry building interfaces and request memb_gather_enter
*/
qb_loop_timer_add (instance->totemudpu_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
} else {
/*
* Interface is up
*/
instance->netif_bind_state = BIND_STATE_REGULAR;
bind_address = &instance->totem_interface->bindnet;
}
/*
* Create and bind the multicast and unicast sockets
*/
totemudpu_build_sockets (instance,
bind_address,
&instance->totem_interface->boundto);
qb_loop_poll_add (instance->totemudpu_poll_handle,
QB_LOOP_MED,
instance->token_socket,
POLLIN, instance, net_deliver_fn);
totemip_copy (&instance->my_id, &instance->totem_interface->boundto);
/*
* This reports changes in the interface to the user and totemsrp
*/
if (instance->netif_bind_state == BIND_STATE_REGULAR) {
if (instance->netif_state_report & NETIF_STATE_REPORT_UP) {
log_printf (instance->totemudpu_log_level_notice,
"The network interface [%s] is now up.",
totemip_print (&instance->totem_interface->boundto));
instance->netif_state_report = NETIF_STATE_REPORT_DOWN;
instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
}
/*
* Add a timer to check for interface going down in single membership
*/
if (instance->my_memb_entries == 1) {
qb_loop_timer_add (instance->totemudpu_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
}
} else {
if (instance->netif_state_report & NETIF_STATE_REPORT_DOWN) {
log_printf (instance->totemudpu_log_level_notice,
"The network interface is down.");
instance->totemudpu_iface_change_fn (instance->context, &instance->my_id);
}
instance->netif_state_report = NETIF_STATE_REPORT_UP;
}
}
/* Set the socket priority to INTERACTIVE to ensure
that our messages don't get queued behind anything else */
static void totemudpu_traffic_control_set(struct totemudpu_instance *instance, int sock)
{
#ifdef SO_PRIORITY
int prio = 6; /* TC_PRIO_INTERACTIVE */
if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(int))) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"Could not set traffic priority");
}
#endif
}
static int totemudpu_build_sockets_ip (
struct totemudpu_instance *instance,
struct totem_ip_address *bindnet_address,
struct totem_ip_address *bound_to,
int interface_num)
{
struct sockaddr_storage sockaddr;
int addrlen;
int res;
unsigned int recvbuf_size;
unsigned int optlen = sizeof (recvbuf_size);
unsigned int retries = 0;
/*
* Setup unicast socket
*/
instance->token_socket = socket (bindnet_address->family, SOCK_DGRAM, 0);
if (instance->token_socket == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"socket() failed");
return (-1);
}
totemip_nosigpipe (instance->token_socket);
res = fcntl (instance->token_socket, F_SETFL, O_NONBLOCK);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"Could not set non-blocking operation on token socket");
return (-1);
}
/*
* Bind to unicast socket used for token send/receives
* This has the side effect of binding to the correct interface
*/
totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &sockaddr, &addrlen);
while (1) {
res = bind (instance->token_socket, (struct sockaddr *)&sockaddr, addrlen);
if (res == 0) {
break;
}
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"bind token socket failed");
if (++retries > BIND_MAX_RETRIES) {
break;
}
/*
* Wait for a while
*/
(void)poll(NULL, 0, BIND_RETRIES_INTERVAL * retries);
}
if (res == -1) {
return (-1);
}
/*
* the token_socket can receive many messages. Allow a large number
* of receive messages on this socket
*/
recvbuf_size = MCAST_SOCKET_BUFFER_SIZE;
res = setsockopt (instance->token_socket, SOL_SOCKET, SO_RCVBUF,
&recvbuf_size, optlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
"Could not set recvbuf size");
}
return 0;
}
static int totemudpu_build_sockets (
struct totemudpu_instance *instance,
struct totem_ip_address *bindnet_address,
struct totem_ip_address *bound_to)
{
int interface_num;
int interface_up;
int res;
/*
* Determine the ip address bound to and the interface name
*/
res = netif_determine (instance,
bindnet_address,
bound_to,
&interface_up,
&interface_num);
if (res == -1) {
return (-1);
}
totemip_copy(&instance->my_id, bound_to);
res = totemudpu_build_sockets_ip (instance,
bindnet_address, bound_to, interface_num);
if (res == -1) {
/* if we get here, corosync won't work anyway, so better leaving than faking to work */
LOGSYS_PERROR (errno, instance->totemudpu_log_level_error,
"Unable to create sockets, exiting");
exit(EXIT_FAILURE);
}
/* We only send out of the token socket */
totemudpu_traffic_control_set(instance, instance->token_socket);
/*
* Rebind all members to new ips
*/
totemudpu_member_list_rebind_ip(instance);
return res;
}
/*
* Totem Network interface - also does encryption/decryption
* depends on poll abstraction, POSIX, IPV4
*/
/*
* Create an instance
*/
int totemudpu_initialize (
qb_loop_t *poll_handle,
void **udpu_context,
struct totem_config *totem_config,
totemsrp_stats_t *stats,
int interface_no,
void *context,
void (*deliver_fn) (
void *context,
const void *msg,
unsigned int msg_len),
void (*iface_change_fn) (
void *context,
const struct totem_ip_address *iface_address),
void (*target_set_completed) (
void *context))
{
struct totemudpu_instance *instance;
instance = malloc (sizeof (struct totemudpu_instance));
if (instance == NULL) {
return (-1);
}
totemudpu_instance_initialize (instance);
instance->totem_config = totem_config;
instance->stats = stats;
/*
* Configure logging
*/
instance->totemudpu_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
instance->totemudpu_log_level_error = totem_config->totem_logging_configuration.log_level_error;
instance->totemudpu_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
instance->totemudpu_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
instance->totemudpu_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
instance->totemudpu_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
instance->totemudpu_log_printf = totem_config->totem_logging_configuration.log_printf;
/*
* Initialize random number generator for later use to generate salt
*/
instance->crypto_inst = crypto_init (totem_config->private_key,
totem_config->private_key_len,
totem_config->crypto_cipher_type,
totem_config->crypto_hash_type,
instance->totemudpu_log_printf,
instance->totemudpu_log_level_security,
instance->totemudpu_log_level_notice,
instance->totemudpu_log_level_error,
instance->totemudpu_subsys_id);
if (instance->crypto_inst == NULL) {
free(instance);
return (-1);
}
/*
* Initialize local variables for totemudpu
*/
instance->totem_interface = &totem_config->interfaces[interface_no];
memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
instance->totemudpu_poll_handle = poll_handle;
instance->totem_interface->bindnet.nodeid = instance->totem_config->node_id;
instance->context = context;
instance->totemudpu_deliver_fn = deliver_fn;
instance->totemudpu_iface_change_fn = iface_change_fn;
instance->totemudpu_target_set_completed = target_set_completed;
totemip_localhost (AF_INET, &localhost);
localhost.nodeid = instance->totem_config->node_id;
/*
* RRP layer isn't ready to receive message because it hasn't
* initialized yet. Add short timer to check the interfaces.
*/
qb_loop_timer_add (instance->totemudpu_poll_handle,
QB_LOOP_MED,
100*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
totemudpu_start_merge_detect_timeout(instance);
*udpu_context = instance;
return (0);
}
void *totemudpu_buffer_alloc (void)
{
return malloc (FRAME_SIZE_MAX);
}
void totemudpu_buffer_release (void *ptr)
{
return free (ptr);
}
int totemudpu_processor_count_set (
void *udpu_context,
int processor_count)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
instance->my_memb_entries = processor_count;
qb_loop_timer_del (instance->totemudpu_poll_handle,
instance->timer_netif_check_timeout);
if (processor_count == 1) {
qb_loop_timer_add (instance->totemudpu_poll_handle,
QB_LOOP_MED,
instance->totem_config->downcheck_timeout*QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_netif_check_timeout,
&instance->timer_netif_check_timeout);
}
return (res);
}
int totemudpu_recv_flush (void *udpu_context)
{
int res = 0;
return (res);
}
int totemudpu_send_flush (void *udpu_context)
{
int res = 0;
return (res);
}
int totemudpu_token_send (
void *udpu_context,
const void *msg,
unsigned int msg_len)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
return (res);
}
int totemudpu_mcast_flush_send (
void *udpu_context,
const void *msg,
unsigned int msg_len)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
mcast_sendmsg (instance, msg, msg_len, 0);
return (res);
}
int totemudpu_mcast_noflush_send (
void *udpu_context,
const void *msg,
unsigned int msg_len)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
mcast_sendmsg (instance, msg, msg_len, 1);
return (res);
}
extern int totemudpu_iface_check (void *udpu_context)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
timer_function_netif_check_timeout (instance);
return (res);
}
extern void totemudpu_net_mtu_adjust (void *udpu_context, struct totem_config *totem_config)
{
assert(totem_config->interface_count > 0);
totem_config->net_mtu -= crypto_sec_header_size(totem_config->crypto_cipher_type,
totem_config->crypto_hash_type) +
totemip_udpip_header_size(totem_config->interfaces[0].bindnet.family);
}
const char *totemudpu_iface_print (void *udpu_context) {
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
const char *ret_char;
ret_char = totemip_print (&instance->my_id);
return (ret_char);
}
int totemudpu_iface_get (
void *udpu_context,
struct totem_ip_address *addr)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
return (res);
}
int totemudpu_token_target_set (
void *udpu_context,
const struct totem_ip_address *token_target)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int res = 0;
memcpy (&instance->token_target, token_target,
sizeof (struct totem_ip_address));
instance->totemudpu_target_set_completed (instance->context);
return (res);
}
extern int totemudpu_recv_mcast_empty (
void *udpu_context)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
unsigned int res;
struct sockaddr_storage system_from;
struct msghdr msg_recv;
struct pollfd ufd;
int nfds;
int msg_processed = 0;
/*
* Receive datagram
*/
msg_recv.msg_name = &system_from;
msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
msg_recv.msg_iov = &instance->totemudpu_iov_recv;
msg_recv.msg_iovlen = 1;
#ifdef HAVE_MSGHDR_CONTROL
msg_recv.msg_control = 0;
#endif
#ifdef HAVE_MSGHDR_CONTROLLEN
msg_recv.msg_controllen = 0;
#endif
#ifdef HAVE_MSGHDR_FLAGS
msg_recv.msg_flags = 0;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTS
msg_recv.msg_accrights = NULL;
#endif
#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
msg_recv.msg_accrightslen = 0;
#endif
do {
ufd.fd = instance->token_socket;
ufd.events = POLLIN;
nfds = poll (&ufd, 1, 0);
if (nfds == 1 && ufd.revents & POLLIN) {
res = recvmsg (instance->token_socket, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
if (res != -1) {
msg_processed = 1;
} else {
msg_processed = -1;
}
}
} while (nfds == 1);
return (msg_processed);
}
static int totemudpu_create_sending_socket(
void *udpu_context,
const struct totem_ip_address *member)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
int fd;
int res;
unsigned int sendbuf_size;
unsigned int optlen = sizeof (sendbuf_size);
struct sockaddr_storage sockaddr;
int addrlen;
fd = socket (member->family, SOCK_DGRAM, 0);
if (fd == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"Could not create socket for new member");
return (-1);
}
totemip_nosigpipe (fd);
res = fcntl (fd, F_SETFL, O_NONBLOCK);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"Could not set non-blocking operation on token socket");
goto error_close_fd;
}
/*
* These sockets are used to send multicast messages, so their buffers
* should be large
*/
sendbuf_size = MCAST_SOCKET_BUFFER_SIZE;
res = setsockopt (fd, SOL_SOCKET, SO_SNDBUF,
&sendbuf_size, optlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_notice,
"Could not set sendbuf size");
/*
* Fail in setting sendbuf size is not fatal -> don't exit
*/
}
/*
* Bind to sending interface
*/
totemip_totemip_to_sockaddr_convert(&instance->my_id, 0, &sockaddr, &addrlen);
res = bind (fd, (struct sockaddr *)&sockaddr, addrlen);
if (res == -1) {
LOGSYS_PERROR (errno, instance->totemudpu_log_level_warning,
"bind token socket failed");
goto error_close_fd;
}
return (fd);
error_close_fd:
close(fd);
return (-1);
}
int totemudpu_member_add (
void *udpu_context,
const struct totem_ip_address *member)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
struct totemudpu_member *new_member;
new_member = malloc (sizeof (struct totemudpu_member));
if (new_member == NULL) {
return (-1);
}
memset(new_member, 0, sizeof(*new_member));
log_printf (LOGSYS_LEVEL_NOTICE, "adding new UDPU member {%s}",
totemip_print(member));
list_init (&new_member->list);
list_add_tail (&new_member->list, &instance->member_list);
memcpy (&new_member->member, member, sizeof (struct totem_ip_address));
new_member->fd = totemudpu_create_sending_socket(udpu_context, member);
new_member->active = 0;
return (0);
}
int totemudpu_member_remove (
void *udpu_context,
const struct totem_ip_address *token_target)
{
int found = 0;
struct list_head *list;
struct totemudpu_member *member;
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
/*
* Find the member to remove and close its socket
*/
for (list = instance->member_list.next;
list != &instance->member_list;
list = list->next) {
member = list_entry (list,
struct totemudpu_member,
list);
if (totemip_compare (token_target, &member->member)==0) {
log_printf(LOGSYS_LEVEL_NOTICE,
"removing UDPU member {%s}",
totemip_print(&member->member));
if (member->fd > 0) {
log_printf(LOGSYS_LEVEL_DEBUG,
"Closing socket to: {%s}",
totemip_print(&member->member));
qb_loop_poll_del (instance->totemudpu_poll_handle,
member->fd);
close (member->fd);
}
found = 1;
break;
}
}
/*
* Delete the member from the list
*/
if (found) {
list_del (list);
}
instance = NULL;
return (0);
}
int totemudpu_member_list_rebind_ip (
void *udpu_context)
{
struct list_head *list;
struct totemudpu_member *member;
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
for (list = instance->member_list.next;
list != &instance->member_list;
list = list->next) {
member = list_entry (list,
struct totemudpu_member,
list);
if (member->fd > 0) {
close (member->fd);
}
member->fd = totemudpu_create_sending_socket(udpu_context, &member->member);
}
return (0);
}
int totemudpu_member_set_active (
void *udpu_context,
const struct totem_ip_address *member_ip,
int active)
{
struct list_head *list;
struct totemudpu_member *member;
int addr_found = 0;
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
/*
* Find the member to set active flag
*/
for (list = instance->member_list.next; list != &instance->member_list; list = list->next) {
member = list_entry (list, struct totemudpu_member, list);
if (totemip_compare (member_ip, &member->member) == 0) {
log_printf(LOGSYS_LEVEL_DEBUG,
"Marking UDPU member %s %s",
totemip_print(&member->member),
(active ? "active" : "inactive"));
member->active = active;
addr_found = 1;
break;
}
}
if (!addr_found) {
log_printf(LOGSYS_LEVEL_DEBUG,
"Can't find UDPU member %s (should be marked as %s)",
totemip_print(member_ip),
(active ? "active" : "inactive"));
}
return (0);
}
static void timer_function_merge_detect_timeout (
void *data)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)data;
if (instance->merge_detect_messages_sent_before_timeout == 0) {
instance->send_merge_detect_message = 1;
}
instance->merge_detect_messages_sent_before_timeout = 0;
totemudpu_start_merge_detect_timeout(instance);
}
static void totemudpu_start_merge_detect_timeout(
void *udpu_context)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
qb_loop_timer_add(instance->totemudpu_poll_handle,
QB_LOOP_MED,
instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
(void *)instance,
timer_function_merge_detect_timeout,
&instance->timer_merge_detect_timeout);
}
static void totemudpu_stop_merge_detect_timeout(
void *udpu_context)
{
struct totemudpu_instance *instance = (struct totemudpu_instance *)udpu_context;
qb_loop_timer_del(instance->totemudpu_poll_handle,
instance->timer_merge_detect_timeout);
}

File Metadata

Mime Type
text/x-diff
Expires
Tue, Feb 25, 2:55 PM (1 d, 5 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1464869
Default Alt Text
(105 KB)

Event Timeline