diff --git a/cts/lxc_autogen.sh.in b/cts/lxc_autogen.sh.in index d06ba2ddb5..da4b92e98b 100755 --- a/cts/lxc_autogen.sh.in +++ b/cts/lxc_autogen.sh.in @@ -1,424 +1,526 @@ #!/bin/bash containers="2" download=0 share_configs=0 # different than default libvirt network in case this is run nested in a KVM instance addr="192.168.123.1" restore=0 restore_pcmk=0 restore_all=0 generate=0 key_gen=0 cib=0 anywhere=0 add_master=0 verify=0 working_dir="@CRM_CONFIG_CTS@/lxc" -curdir=$(pwd) run_dirs="/run /var/run /usr/var/run" +SSH_CMD_OPTS=" + -o StrictHostKeyChecking=no + -o ConnectTimeout=30 + -o BatchMode=yes + -l root + -T +" +# must be on one line b/c used inside quotes +SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no" + function helptext() { echo "lxc_autogen.sh - A tool for generating libvirt lxc containers for testing purposes." echo "" echo "Usage: lxc-autogen [options]" echo "" echo "Options:" echo "-g, --generate Generate libvirt lxc environment in the directory this script is run from." - echo "-k, --key-gen Generate local pacemaker remote key only." + echo "-k, --key-gen Generate pacemaker remote key only." echo "-r, --restore-libvirt Restore the default network, and libvirt config to before this script ran." echo "-p, --restore-cib Remove cib entries this script generated." echo "-R, --restore-all Restore both libvirt and cib plus clean working directory. This will leave libvirt xml files though so rsc can be stopped properly." echo "" echo "-A, --allow-anywhere Allow the containers to live anywhere in the cluster" echo "-a, --add-cib Add remote-node entries for each lxc instance into the cib" echo "-m, --add-master Add master resource shared between remote-nodes" echo "-d, --download-agent Download and install the latest VirtualDomain agent." - echo "-s, --share-configs Copy container configs to all other known cluster nodes, (crm_node -l)" + echo "-s, --share-configs Synchronize on all known cluster nodes" echo "-c, --containers Specify the number of containers to generate, defaults to $containers. Used with -g" echo "-n, --network What network to override default libvirt network to. Example: -n 192.168.123.1. Used with -g" echo "-v, --verify Verify environment is capable of running lxc" echo "" exit $1 } while true ; do case "$1" in --help|-h|-\?) helptext 0;; -c|--containers) containers="$2"; shift; shift;; -d|--download-agent) download=1; shift;; -s|--share-configs) share_configs=1; shift;; -n|--network) addr="$2"; shift; shift;; -r|--restore-libvirt) restore=1; shift;; -p|--restore-cib) restore_pcmk=1; shift;; -R|--restore-all) restore_all=1 restore=1 restore_pcmk=1 shift;; - -g|--generate) generate=1; shift;; + -g|--generate) generate=1; key_gen=1; shift;; -k|--key-gen) key_gen=1; shift;; -a|--add-cib) cib=1; shift;; -A|--allow-anywhere) anywhere=1; shift;; -m|--add-master) add_master=1; shift;; -v|--verify) verify=1; shift;; "") break;; *) helptext 1;; esac done if [ $verify -eq 1 ]; then # verify virsh tool is available and that # we can connect to lxc driver. virsh -c lxc:/// list --all > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "Could not connect 'virsh -c lxc:///' check that libvirt lxc driver is installed" # yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell exit 1 fi cat /etc/selinux/config | grep -e "SELINUX.*=.*permissive" -e "SELINUX.*=.*enforcing" > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "/etc/selinux/config must have SELINUX set to permissive or enforcing mode." exit 1 fi ps x > /tmp/lxc-autogen-libvirt-test.txt grep "libvirtd" /tmp/lxc-autogen-libvirt-test.txt if [ $? -ne 0 ]; then rm -f /tmp/lxc-autogen-libvirt-test.txt echo "libvirtd isn't up." exit 1 fi rm -f /tmp/lxc-autogen-libvirt-test.txt which rsync > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "rsync is required" fi which pacemaker_remoted > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "pacemaker_remoted is required" fi fi #strip last digits off addr addr=$(echo $addr | awk -F. '{print $1"."$2"."$3}') +this_node() +{ + crm_node -n +} + +other_nodes() +{ + crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}" +} + +make_directory() +{ + # argument must be full path + DIR="$1" + + mkdir -p "$DIR" + if [ $share_configs -eq 1 ]; then + for node in $(other_nodes); do + ssh $SSH_CMD_OPTS $node mkdir -p "$DIR" + done + fi +} + +sync_file() +{ + TARGET="$1" + + if [ $share_configs -eq 1 ]; then + for node in $(other_nodes); do + rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}" + done + fi +} + +download_agent() +{ + wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain + chmod 755 VirtualDomain + mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain + sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain +} + set_network() { rm -f cur_network.xml cat << END >> cur_network.xml default 41ebdb84-7134-1111-a136-91f0f1119225 END + sync_file ${working_dir}/cur_network.xml +} + +distribute_configs() +{ + for node in $(other_nodes); do + rsync -ave "ssh $SSH_RSYNC_OPTS" ${working_dir}/lxc*.xml ${node}:${working_dir} + rsync -ave "ssh $SSH_RSYNC_OPTS" ${working_dir}/lxc*-filesystem ${node}:${working_dir} + done +} + +start_network() +{ + NODE="$1" + ssh $SSH_CMD_OPTS $NODE <<-EOF + cd $working_dir virsh net-info default >/dev/null 2>&1 - if [ $? -eq 0 ]; then + if [ \$? -eq 0 ]; then if [ ! -f restore_default.xml ]; then virsh net-dumpxml default > restore_default.xml fi virsh net-destroy default virsh net-undefine default fi virsh net-define cur_network.xml virsh net-start default virsh net-autostart default +EOF +} + +start_network_all() +{ + start_network $(this_node) + if [ $share_configs -eq 1 ]; then + for node in $(other_nodes); do + start_network $node + done + fi +} + +add_hosts_entry() +{ + IP="$1" + HNAME="$2" + + echo $IP $HNAME >>/etc/hosts + if [ $share_configs -eq 1 ]; then + for node in $(other_nodes); do + ssh $SSH_CMD_OPTS $node "echo $IP $HNAME >>/etc/hosts" + done + fi } generate_key() { - #generate pacemaker remote key - ls /etc/pacemaker/authkey > /dev/null 2>&1 - if [ $? != 0 ]; then - mkdir -p /etc/pacemaker - dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 + if [ ! -e /etc/pacemaker/authkey ]; then + make_directory /etc/pacemaker + dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 + sync_file /etc/pacemaker/authkey fi } generate() { set_network # Generate libvirt domains in xml for (( c=1; c <= $containers; c++ )) do - rm -rf lxc$c-filesystem + # Clean any previous definition + rm -rf lxc$c.xml lxc$c-filesystem + + # Create a basic filesystem with run directories for dir in $run_dirs; do mkdir -p lxc$c-filesystem/$dir done - rm -f lxc$c.xml + # Create libvirt definition suffix=$((10 + $c)) prefix=$(echo $addr | awk -F. '{print $1"."$2}') subnet=$(echo $addr | awk -F. '{print $3}') while [ $suffix -gt 255 ]; do subnet=$(($subnet + 1)) suffix=$(($subnet - 255)) done + cip=$prefix.$subnet.$suffix cat << END >> lxc$c.xml lxc$c 200704 exe $working_dir/lxc$c-filesystem/launch-helper END for dir in $run_dirs; do cat << END >> lxc$c.xml - + END done cat << END >> lxc$c.xml END + + # Create CIB definition rm -f container$c.cib cat << END >> container$c.cib END + # Create container init rm -f lxc$c-filesystem/launch-helper cat << END >> lxc$c-filesystem/launch-helper #!/bin/bash -ip -f inet addr add $prefix.$subnet.$suffix/24 dev eth0 +ip -f inet addr add $cip/24 dev eth0 ip link set eth0 up ip route add default via $addr.1 hostname lxc$c df > $working_dir/lxc$c-filesystem/disk_usage.txt export PCMK_debugfile=/var/log/pacemaker_remote_lxc$c.log /usr/sbin/pacemaker_remoted END chmod 711 lxc$c-filesystem/launch-helper - cat << END >> /etc/hosts -$prefix.$subnet.$suffix lxc$c -END + add_hosts_entry $cip lxc$c done + # Create CIB fragment for a master-slave resource rm -f lxc-ms.cib cat << END >> lxc-ms.cib END } apply_cib_master() { cibadmin -Q > cur.cib export CIB_file=cur.cib cibadmin -o resources -Mc -x lxc-ms.cib - for tmp in $(ls lxc*.xml); do - tmp=$(echo $tmp | sed -e 's/\.xml//g') + for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do echo "" > tmp_constraint cibadmin -o constraints -Mc -x tmp_constraint done # Make sure the version changes even if the content doesn't cibadmin -B unset CIB_file cibadmin --replace -o configuration --xml-file cur.cib rm -f cur.cib } apply_cib_entries() { - node=$(crm_node -n) - cibadmin -Q > cur.cib export CIB_file=cur.cib for tmp in $(ls container*.cib); do cibadmin -o resources -Mc -x $tmp remote_node=$(cat ${tmp} | grep remote-node | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p') if [ $anywhere -eq 0 ]; then tmp=$(echo $tmp | sed -e 's/\.cib//g') - crm_resource -M -r $tmp -H $node + crm_resource -M -r $tmp -H $(this_node) fi echo "" > tmp_constraint # it's fine if applying this constraint fails. it's just to help with cts # when the connectivity resources are in use. those resources fail the remote-nodes. cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1 for rsc in $(crm_resource -l | grep rsc_ ); do echo "" > tmp_constraint cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1 done rm -f tmp_constraint done # Make sure the version changes even if the content doesn't cibadmin -B unset CIB_file cibadmin --replace -o configuration --xml-file cur.cib rm -f cur.cib } restore_cib() { - node=$(crm_node -n) cibadmin -Q > cur.cib export CIB_file=cur.cib - for tmp in $(ls lxc*.xml); do - tmp=$(echo $tmp | sed -e 's/\.xml//g') + for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do echo "" > tmp_constraint cibadmin -o constraints -D -x tmp_constraint echo "" > tmp_constraint cibadmin -o constraints -D -x tmp_constraint for rsc in $(crm_resource -l | grep rsc_ ); do echo "" > tmp_constraint cibadmin -o constraints -D -x tmp_constraint done rm -f tmp_constraint done cibadmin -o resources -D -x lxc-ms.cib for tmp in $(ls container*.cib); do tmp=$(echo $tmp | sed -e 's/\.cib//g') - crm_resource -U -r $tmp -H $node + crm_resource -U -r $tmp -H $(this_node) crm_resource -D -r $tmp -t primitive done # Make sure the version changes even if the content doesn't cibadmin -B unset CIB_file cibadmin --replace -o configuration --xml-file cur.cib rm -f cur.cib # Allow the cluster to stabilize before continuing crm_resource --wait # Purge nodes from caches and CIB status section - for tmp in $(ls lxc*.xml); do - tmp=$(echo $tmp | sed -e 's/\.xml//g') + for tmp in $(ls lxc*.xml | sed -e 's/\.xml//g'); do crm_node --force --remove $tmp done } -restore_libvirt() +restore_network() { - for tmp in $(ls lxc*.xml); do - tmp=$(echo $tmp | sed -e 's/\.xml//g') - virsh -c lxc:/// destroy $tmp > /dev/null 2>&1 - virsh -c lxc:/// undefine $tmp > /dev/null 2>&1 - - sed -i.bak "/...\....\....\..* ${tmp}/d" /etc/hosts - echo "$tmp destroyed" + NODE="$1" + + ssh $SSH_CMD_OPTS $NODE <<-EOF + cd $working_dir + for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do + virsh -c lxc:/// destroy \$tmp >/dev/null 2>&1 + virsh -c lxc:/// undefine \$tmp >/dev/null 2>&1 + sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts done - - virsh net-destroy default > /dev/null 2>&1 - virsh net-undefine default > /dev/null 2>&1 + virsh net-destroy default >/dev/null 2>&1 + virsh net-undefine default >/dev/null 2>&1 if [ -f restore_default.xml ]; then virsh net-define restore_default.xml virsh net-start default - if [ $? -eq 0 ]; then - echo "default network restored" - fi + rm restore_default.xml fi - rm -f restore_default.xml > /dev/null 2>&1 +EOF + echo "Containers destroyed and default network restored on $NODE" } -distribute_configs() +restore_libvirt() +{ + restore_network $(this_node) + if [ $share_configs -eq 1 ]; then + for node in $(other_nodes); do + restore_network $node + done + fi +} + +restore_files() { - nodes=`crm_node -l | awk '{print $2}'` - for node in $nodes; do - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o BatchMode=yes -l root $node mkdir -p /$working_dir - rsync -ave 'ssh -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no' $working_dir/lxc*.xml $node:/$working_dir - rsync -ave 'ssh -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no' $working_dir/lxc*-filesystem $node:/$working_dir - done + ls | grep -v "lxc.\.xml" | xargs rm -rf + if [ $share_configs -eq 1 ]; then + for node in $(other_nodes); do + ssh $SSH_CMD_OPTS $node rm -rf \ + $working_dir/lxc*-filesystem \ + $working_dir/cur_network.xml + done + fi } -mkdir -p $working_dir +make_directory $working_dir cd $working_dir +# Generate files as requested if [ $download -eq 1 ]; then - wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain - chmod 755 VirtualDomain - mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain -fi -if [ $restore_pcmk -eq 1 ]; then - restore_cib -fi -if [ $restore -eq 1 ]; then - restore_libvirt + download_agent fi if [ $key_gen -eq 1 ]; then generate_key fi if [ $generate -eq 1 ]; then - if [ $key_gen -eq 0 ]; then - generate_key - fi generate fi +if [ $share_configs -eq 1 ]; then + distribute_configs +fi +if [ $generate -eq 1 ]; then + start_network_all +fi + +# Update cluster as requested if [ $cib -eq 1 ]; then apply_cib_entries fi if [ $add_master -eq 1 ]; then apply_cib_master fi -if [ $share_configs -eq 1 ]; then - distribute_configs + +# Restore original state as requested +if [ $restore_pcmk -eq 1 ]; then + restore_cib +fi +if [ $restore -eq 1 ]; then + restore_libvirt fi if [ $restore_all -eq 1 ]; then - ls | grep -v "lxc.\.xml" | xargs rm -rf + restore_files fi - -cd $curdir