diff --git a/extra/cluster-init b/extra/cluster-init index af7d02c73e..1e4fdc67b0 100755 --- a/extra/cluster-init +++ b/extra/cluster-init @@ -1,806 +1,808 @@ #!/bin/bash accept_defaults=0 do_raw=0 ETCHOSTS=0 CMAN=0 do_heartbeat=0 plugin_ver=-1 pcmk_ver=11 nodelist=0 pkgs="corosync xinetd nmap abrt-cli fence-agents perl-TimeDate gdb" transport="multicast" +inaddr_any="no" INSTALL= cs_conf= fence_conf= rpm_repo= distro= dsh_group=0 cluster=dummy0 # Corosync/OpenAIS Settings cs_port=666 # Settings that work great on nXX join=60 #token=3000 consensus=1500 # Official settings join=2000 token=5000 consensus=2500 # Testing join=1000 consensus=7500 do_debug=off function ip_for_node() { ping -c 1 $1 | grep "bytes from" | head -n 1 | sed -e 's/.*bytes from//' -e 's/: icmp.*//' | awk '{print $NF}' | sed 's:(::' | sed 's:)::' # if [ $do_raw = 1 ]; then # echo $1 # else # #host $1 | grep "has address" | head -n 1 | awk '{print $NF}' | sed 's:(::' | sed 's:)::' # fi } function id_for_node() { ip_for_node $* | tr '.' ' ' | awk '{print $4}' } function name_for_node() { echo $1 | awk -F. '{print $1}' } function helptext() { echo "cluster-init - Configure cluster communication for the different infrastructures supported by Pacemaker" echo "" echo "-g, --group Specify the group to operate on/with" echo "-w, --host Specify a host to operate on/with. May be specified multiple times" echo "-r, --raw-ip Supplied nodes were listed as their IP addresses" echo "" echo "-h, --heartbeat configure for heartbeat" echo "-c, --corosync configure for corosync" echo "-C, --nodelist configure for corosync with a node list" echo "-o, --openais configure for openais" echo "--cman configure for cman" echo "-p, --plugin version" echo "-u, --unicast configure point-to-point communication instead of multicast (corosync only)" echo "" echo "-I, --install Install packages" echo "-R, --repo name Setup and update/install Pacemaker from the named clusterlabs.org repo" echo " Known values: rpm, rpm-test, rpm-next, rpm-test-next, rpm-test-rhel" echo "-D, --distro The distro within the --repo. Defaults to fedora-15" echo "" echo "-d, --debug Enable debug logging for the cluster" echo "-10 install stable-1.0 packages, implies: -p 0 -R rpm-test -I" echo "--hosts Copy /etc/hosts from the test master to the nodes" echo "-e, --extra package-list" echo " Extra packages to install" exit $1 } host_input="" while true; do case "$1" in -g) cluster=$2; dsh_group=`echo $cluster | sed s/[a-zA-Z]*//` host_input="-g $cluster" shift; shift;; -w|--host) for h in $2; do host_input="$host_input -w $h"; done shift; shift;; -w) host_input="$host_input -w $2" shift; shift;; -r|--raw-ip) do_raw=1; shift;; -D) distro=$2; shift; shift;; -d|--debug) do_debug=on; shift;; -R|--repo) rpm_repo=$2; shift; shift;; -I|--install) INSTALL=Yes; shift;; --hosts) ETCHOSTS=1; shift;; cman|--cman) CTYPE=cman; shift;; -h|--heartbeat) CTYPE=heartbeat; shift;; -c|--corosync) CTYPE=corosync; shift;; -C|--nodelist) CTYPE=corosync; nodelist=1; shift;; -o|--openais) CTYPE=openais; shift;; --plugin|-p) CTYPE=plugin; plugin_ver=$2; shift; shift;; - -u|--unicast) nodelist=1; transport=udpu; shift;; + -u|--unicast) nodelist=1; transport=udpu; inaddr_any="yes"; shift;; -e|--extra) pkgs="$pkgs $2"; shift; shift;; -t|--test) rpm_repo=rpm-test-next; pkgs="$pkgs valgrind"; shift;; r*[0-9]) rhel=`echo $1 | sed -e s/rhel// -e s/-// -e s/r//` distro="rhel-$rhel"; pkgs="$pkgs qarsh-server"; case $rhel in 6) CTYPE=cman;; 7) CTYPE=corosync;; esac shift ;; f*[0-9][0-9]) distro="fedora-`echo $1 | sed -e s/fedora// -e s/-// -e s/f//`"; CTYPE=corosync; shift ;; p0|10) pcmk_ver=10; plugin_ver=0; rpm_repo="rpm-test"; install=1; shift;; -y|--yes|--defaults) accept_defaults=1; shift;; -x) set -x; shift;; -\?|--help) helptext 0; shift;; "") break;; *) echo "unknown option: $1"; exit 1;; esac done if [ x = "x$host_input" -a x = "x$CTS_GROUP" ]; then if [ -d $HOME/.dsh/group ]; then read -p "Please specify a dsh group you'd like to configure as a cluster? [] " -t 60 CTS_GROUP else read -p "Please specify a whitespace delimetered list of nodes you'd like to configure as a cluster? [] " -t 60 host_list for h in $2; do host_input="$host_input -w $h"; done fi fi if [ ! -z $CTS_GROUP ]; then host_input="-g $CTS_GROUP" dsh_group=`echo $CTS_GROUP | sed s/[a-zA-Z]*//` fi if [ -z "$host_input" ]; then echo "You didn't specify any nodes or groups to configure" exit 1 fi host_list=`cluster-helper --list short $host_input` num_hosts=`echo $host_list | wc -w` if [ -z $dsh_group ]; then dsh_group=1 fi if [ -z $CTYPE ]; then echo "" read -p "Where should Pacemaker obtain membership and quorum from? [corosync] (corosync, cman, plugin, openais, heartbeat) " -t 60 CTYPE fi case $CTYPE in cman) CMAN=1; cs_conf=/etc/cluster/cluster.conf;; corosync) cs_conf=/etc/corosync/corosync.conf;; heartbeat) do_heartbeat=1;; openais) cs_conf=/etc/ais/openais.conf;; plugin) cs_conf=/etc/corosync/corosync.conf if [ -z $plugin_ver ]; then echo "" read -p "Should corosync start Pacemaker? [No]" -t 60 PTYPE case $PTYPE in [Yy][Ee][Ss]|[Yy]) plugin_ver=0;; *) plugin_ver=1;; esac fi ;; esac function get_defaults() { if [ -z $SSH ]; then SSH="No" fi if [ -z $SELINUX ]; then SELINUX="Yes" fi if [ -z $IPTABLES ]; then IPTABLES="Yes" fi if [ -z $DOMAIN ]; then DOMAIN="No" fi if [ -z $INSTALL ]; then INSTALL="Yes" fi if [ -z $DATE ]; then DATE="Yes" fi } get_defaults if [ $accept_defaults = 0 ]; then echo "" read -p "Shall I install an ssh key to cluster nodes? [$SSH] " -t 60 SSH echo "" echo "SELinux prevent many things, including password-less ssh logins" read -p "Shall I disable selinux? [$SELINUX] " -t 60 SELINUX echo "" echo "Incorrectly configured firewalls will prevent corosync from starting up" read -p "Shall I disable iptables? [$IPTABLES] " -t 60 IPTABLES if [ $CMAN = 1 ]; then echo "" echo "Without a default domain, cman probably wont start because it can't look up the IP for hosts" read -p "Shall I set one? [No] (domain.name) " -t 60 DOMAIN elif [ $pcmk_ver = 10 ]; then echo "" echo "Without a default domain, external/ssh fencing probably wont work because it can't find its peers" read -p "Shall I set one? [No] (domain.name) " -t 60 DOMAIN fi echo "" read -p "Shall I install/update the relevant packages? [$INSTALL] " -t 60 INSTALL case $INSTALL in [Yy][Ee][Ss]|[Yy]|"") if [ -z $rpm_repo ]; then echo "" read -p "Would you like to install packages from ClusterLabs.org? [No] (rpm, rpm-next, rpm-test-next) " -t 60 rpm_repo fi if [ ! -z $rpm_repo ]; then if [ -z $distro ]; then distro=fedora-18 echo "" read -p "Which distro are you installing for? [$distro] (eg. fedora-17, rhel-6) " -t 60 distro fi fi ;; esac echo "" read -p "Shall I sync the date/time? [$DATE] " -t 60 DATE fi get_defaults echo "" echo "Detecting possible fencing options" if [ -e /etc/cluster/fence_xvm.key ]; then echo "* Found fence_xvm" fence_conf=/etc/cluster/fence_xvm.key pkgs="$pkgs fence-virt" fi if [ ! -z ${OS_AUTH_URL} ]; then echo "* Found openstack credentials" fence_conf=/sbin/fence_openstack pkgs="$pkgs python-novaclient" fi echo "" echo "Beginning cluster configuration" echo "" case $SSH in [Yy][Ee][Ss]|[Yy]) for host in $host_list; do echo "Installing our ssh key on ${host}" ssh-copy-id root@${host} >/dev/null 2>&1 # Fix selinux labeling ssh -l root ${host} -- restorecon -R -v . done ;; esac case $DATE in [Yy][Ee][Ss]|[Yy]) now=`date` for host in $host_list; do echo "Setting time on ${host}" scp /etc/localtime root@${host}:/etc ssh -l root ${host} -- date -s "'$now'" echo "" done ;; esac REPO= if [ ! -z $rpm_repo ]; then REPO=$rpm_repo/$distro fi if [ $do_heartbeat = 1 ]; then pkgs="$pkgs heartbeat" fi if [ $CMAN = 1 ]; then pkgs="$pkgs cman" fi init=`mktemp` cat<<-END>$init verbose=0 pkgs="$pkgs" lhost=\`uname -n\` lshort=\`echo \$lhost | awk -F. '{print \$1}'\` log() { printf "%-10s \$*\n" "\$lshort:" 1>&2 } debug() { if [ \$verbose -gt 0 ]; then log "Debug: \$*" fi } info() { log "\$*" } warning() { log "WARN: \$*" } fatal() { log "ERROR: \$*" exit 1 } case $SELINUX in [Yy][Ee][Ss]|[Yy]|"") sed -i.sed "s/enforcing/disabled/g" /etc/selinux/config ;; esac case $IPTABLES in [Yy][Ee][Ss]|[Yy]|"") service iptables stop chkconfig iptables off service firewalld stop chkconfig firewalld off ;; esac case $DOMAIN in [Nn][Oo]|"") ;; *.*) if ! grep domain /etc/resolv.conf then sed -i.sed "s/nameserver/domain\ $DOMAIN\\\nnameserver/g" /etc/resolv.conf fi ;; *) echo "Unknown domain: $DOMAIN";; esac case $INSTALL in [Yy][Ee][Ss]|[Yy]|"") if [ ! -z $REPO ]; then info Configuring Clusterlabs repo: $REPO yum install -y wget rm -f /etc/yum.repos.d/clusterlabs.repo wget -O /etc/yum.repos.d/clusterlabs.repo http://www.clusterlabs.org/$REPO/clusterlabs.repo &>/dev/null yum clean all fi info Installing cluster software if [ $pcmk_ver = 10 ]; then yum install -y $pkgs at service atd start systemctl enable atd.service yum install -y "pacemaker < 1.1" else yum install -y $pkgs pacemaker fi ;; esac info "Configuring services" chkconfig xinetd on service xinetd start &>/dev/null chkconfig corosync off &> /dev/null chkconfig heartbeat off &> /dev/null mkdir -p /etc/cluster info "Turning on core files" grep -q "unlimited" /etc/bashrc if [ $? = 1 ]; then sed -i.sed "s/bashrc/bashrc\\\nulimit\ -c\ unlimited/g" /etc/bashrc fi if [ $CMAN = 1 ]; then grep -q "logger" /etc/init.d/cman if [ $? = 1 ]; then info "Turning on CMAN init-script logging" sed -i.sed "s/function=/logger\ -p\ daemon.info\ cman\ init\ \\\$1\\\nfunction=/g" /etc/init.d/cman fi sed -i.sed "s/.*CMAN_QUORUM_TIMEOUT=.*/CMAN_QUORUM_TIMEOUT=0/g" /etc/sysconfig/cman fi END function create_hb_config() { cat <<-END >/tmp/lha.$$ auth 1 1 crc END printf "%-10s Installing authkeys\n" ${host} scp -q /tmp/lha.$$ root@${host}:/etc/ha.d/authkeys ssh -l root ${host} -- chmod 600 /etc/ha.d/authkeys cat <<-END >/tmp/lha.$$ traditional_compression off compression bz2 realtime yes conn_logd_time 120 coredumps true udpport $cs_port$dsh_group bcast eth0 autojoin any logfacility daemon crm respawn # 8-node version debug 0 keepalive 1 warntime 6 deadtime 10 initdead 15 END printf "%-10s Installing ha.cf\n" ${host} scp -q /tmp/lha.$$ root@${host}:/etc/ha.d/ha.cf } function patch_cs_config() { test $num_hosts != 2 two_node=$? priority="info" if [ $do_debug = 1 ]; then priority="debug" fi ssh -l root ${host} -- sed -i.sed "s/.*mcastaddr:.*/mcastaddr:\ 226.94.1.1/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/.*mcastport:.*/mcastport:\ $cs_port$dsh_group/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/.*bindnetaddr:.*/bindnetaddr:\ $ip/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/.*syslog_facility:.*/syslog_facility:\ daemon/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/.*logfile_priority:.*/logfile_priority:\ $priority/g" $cs_conf if [ ! -z $token ]; then ssh -l root ${host} -- sed -i.sed "s/.*token:.*/token:\ $token/g" $cs_conf fi if [ ! -z $consensus ]; then ssh -l root ${host} -- sed -i.sed "s/.*consensus:.*/consensus:\ $consensus/g" $cs_conf fi if [ ! -z $join ]; then ssh -l root ${host} -- sed -i.sed "s/^join:.*/join:\ $join/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/\\\Wjoin:.*/join:\ $join/g" $cs_conf fi if [ $plugin_ver -ge 0 ]; then ssh -l root ${host} -- grep -q "name: pacemaker" $cs_conf 2>&1 > /dev/null if [ $? = 0 ]; then ssh -l root ${host} -- sed -i.sed "s/.*ver:.*/ver:\ $plugin_ver/g" $cs_conf else printf "%-10s Wrong quorum provider: installing $cs_conf for plugin v${plugin_ver} instead\n" ${host} create_cs_config fi elif [ $CMAN = 1 ]; then ssh -l root ${host} -- grep -q "quorum_cman" $cs_conf 2>&1 > /dev/null if [ $? = 0 ]; then ssh -l root ${host} -- sed -i.sed "s/\\\Wexpected_votes:.*/expected_votes:\ $num_hosts/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/\\\Wtwo_node:.*/two_node:\ $two_node/g" $cs_conf else printf "%-10s Wrong quorum provider: installing $cs_conf for cman instead\n" ${host} create_cs_config fi else ssh -l root ${host} -- grep -q "corosync_votequorum" $cs_conf 2>&1 > /dev/null if [ $? = 0 ]; then ssh -l root ${host} -- sed -i.sed "s/\\\Wexpected_votes:.*/expected_votes:\ $num_hosts/g" $cs_conf ssh -l root ${host} -- sed -i.sed "s/\\\Wtwo_node:.*/two_node:\ $two_node/g" $cs_conf else printf "%-10s Wrong quorum provider: installing $cs_conf for corosync instead\n" ${host} create_cs_config fi fi } function create_cs_config() { cs_tmp=/tmp/cs_conf.$$ test $num_hosts != 2 two_node=$? # Base config priority="info" if [ $do_debug = 1 ]; then priority="debug" fi cat <<-END >$cs_tmp # Please read the corosync.conf.5 manual page totem { version: 2 # cypto_cipher and crypto_hash: Used for mutual node authentication. # If you choose to enable this, then do remember to create a shared # secret with "corosync-keygen". crypto_cipher: none crypto_hash: none # Assign a fixed node id nodeid: $id # Disable encryption secauth: off transport: $transport + inaddr_any: $inaddr_any # interface: define at least one interface to communicate # over. If you define more than one interface stanza, you must # also set rrp_mode. interface { # Rings must be consecutively numbered, starting at 0. ringnumber: 0 # This is normally the *network* address of the # interface to bind to. This ensures that you can use # identical instances of this configuration file # across all your cluster nodes, without having to # modify this option. bindnetaddr: $ip # However, if you have multiple physical network # interfaces configured for the same subnet, then the # network address alone is not sufficient to identify # the interface Corosync should bind to. In that case, # configure the *host* address of the interface # instead: # bindnetaddr: 192.168.1.1 # When selecting a multicast address, consider RFC # 2365 (which, among other things, specifies that # 239.255.x.x addresses are left to the discretion of # the network administrator). Do not reuse multicast # addresses across multiple Corosync clusters sharing # the same network. # Corosync uses the port you specify here for UDP # messaging, and also the immediately preceding # port. Thus if you set this to 5405, Corosync sends # messages over UDP ports 5405 and 5404. mcastport: $cs_port$dsh_group # Time-to-live for cluster communication packets. The # number of hops (routers) that this ring will allow # itself to pass. Note that multicast routing must be # specifically enabled on most network routers. ttl: 1 mcastaddr: 226.94.1.1 } } logging { debug: off fileline: off to_syslog: yes to_stderr: no syslog_facility: daemon timestamp: on to_logfile: yes logfile: /var/log/corosync.log logfile_priority: $priority } amf { mode: disabled } END # Corosync Variant if [ $plugin_ver -ge 0 ]; then cat <<-END >>$cs_tmp service { name: pacemaker ver: $plugin_ver } END elif [ $CMAN = 1 ]; then cat <<-END >>$cs_tmp cluster { name: $cluster clusternodes { END for peer in $host_list; do p_name=`name_for_node $peer` p_id=`id_for_node $peer` cat <<-END >>$cs_tmp clusternode { votes: 1 nodeid: $p_id name: $p_name } END done cat <<-END >>$cs_tmp } cman { expected_votes: $num_hosts cluster_id: $dsh_group nodename: $cs_short_host two_node: $two_node max_queued: 10 } } service { name: corosync_cman ver: 0 } quorum { provider: quorum_cman } END elif [ $nodelist = 1 ]; then cat <<-END >>$cs_tmp nodelist { END for peer in $host_list; do p_name=`name_for_node $peer` p_id=`id_for_node $peer` p_ip=`ip_for_node $peer` cat <<-END >>$cs_tmp node { ring0_addr: $p_ip nodeid: $p_id quorum_votes: 1 name: $peer } END done cat <<-END >>$cs_tmp } quorum { provider: corosync_votequorum expected_votes: $num_hosts votes: 1 two_node: $two_node wait_for_all: 0 last_man_standing: 0 auto_tie_breaker: 0 } END else cat <<-END >>$cs_tmp quorum { provider: corosync_votequorum expected_votes: $num_hosts votes: 1 two_node: $two_node wait_for_all: 0 last_man_standing: 0 auto_tie_breaker: 0 } END fi scp -q $cs_tmp root@${host}:$cs_conf rm -f $cs_tmp } function create_cman_config() { cs_tmp=/tmp/cs_conf.$$ cat <<-END >$cs_tmp END lpc=1 for h in $host_list; do short_h=`name_for_node $h` cat <<-END >>$cs_tmp END lpc=`expr $lpc + 1` done test $num_hosts != 2 two_node=$? extra="" if [ $two_node = 1 ]; then extra="\n " fi cat <<-END >>$cs_tmp $extra END scp -q $cs_tmp root@${host}:$cs_conf rm -f $cs_tmp } for host in $host_list; do echo "" echo "" echo "* Configuring $host" cs_short_host=`name_for_node $host` ip=`ip_for_node $host` id=`id_for_node $host` echo $ip | grep -qis NXDOMAIN if [ $? = 0 ]; then echo "Couldn't find resolve $host to an IP address" exit 1 fi if [ `uname -n` = $host ]; then bash $init else cat $init | ssh -l root -T $host -- "cat > $init; bash $init" fi if [ "x$fence_conf" != x ]; then if [ -e $fence_conf ]; then scp $fence_conf root@${host}:$fence_conf fi fi if [ $ETCHOSTS = 1 ]; then scp /etc/hosts root@${host}:/etc/hosts fi if [ $pcmk_ver = 10 ]; then scp /etc/hosts root@${host}:/etc/hosts scp ~/.ssh/id_dsa.suse root@${host}:.ssh/id_dsa scp ~/.ssh/known_hosts root@${host}:.ssh/known_hosts fi if [ $do_heartbeat = 1 ]; then create_hb_config ${host} elif [ $CMAN = 1 ]; then create_cman_config printf "%-10s Installed $cs_conf\n" ${host}: else ssh -l root ${host} -- grep -q "token:" $cs_conf 2>&1 > /dev/null new_config=$? new_config=1 if [ $new_config = 0 ]; then printf "%-10s Updating $cs_conf\n" ${host}: patch_cs_config else printf "%-10s Installing $cs_conf\n" ${host}: create_cs_config fi fi done