diff --git a/cts/lxc_autogen.sh.in b/cts/lxc_autogen.sh.in index 87d01124be..b758cde1b0 100644 --- a/cts/lxc_autogen.sh.in +++ b/cts/lxc_autogen.sh.in @@ -1,541 +1,545 @@ #!@BASH_PATH@ # # Copyright 2013-2021 the Pacemaker project contributors # # The version control history for this file may have further details. # # This source code is licensed under the GNU General Public License version 2 # or later (GPLv2+) WITHOUT ANY WARRANTY. # containers="2" download=0 share_configs=0 # different than default libvirt network in case this is run nested in a KVM instance addr="192.168.123.1" restore=0 restore_pcmk=0 restore_all=0 generate=0 key_gen=0 cib=0 anywhere=0 add_clone=0 verify=0 working_dir="@CRM_CONFIG_CTS@/lxc" run_dirs="/run /var/run /usr/var/run" # must be on one line b/c used inside quotes SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no" function helptext() { echo "lxc_autogen.sh - generate libvirt LXC containers for testing purposes" echo "" echo "Usage: lxc-autogen [options]" echo "" echo "Options:" echo "-g, --generate Generate libvirt LXC environment in directory this script is run from" echo "-k, --key-gen Generate Pacemaker Remote key only" echo "-r, --restore-libvirt Restore the default network and libvirt config to before this script ran" echo "-p, --restore-cib Remove CIB entries this script generated" echo "-R, --restore-all Restore both libvirt and CIB, and clean working directory" echo " (libvirt xml files are not removed, so resource can be stopped properly)" echo "" echo "-A, --allow-anywhere Allow the containers to live anywhere in the cluster" echo "-a, --add-cib Add CIB entries to create a guest node for each LXC instance" echo "-C, --add-clone Add promotable clone resource shared between LXC guest nodes" echo "-d, --download-agent Download and install latest VirtualDomain agent" echo "-s, --share-configs Synchronize on all known cluster nodes" echo "-c, --containers Specify number of containers to generate (default $containers; used with -g)" echo "-n, --network Network to override libvirt default (example: -n 192.168.123.1; used with -g)" echo "-v, --verify Verify environment is capable of running LXC" echo "" exit "$1" } while true ; do case "$1" in --help|-h|-\?) helptext 0;; -c|--containers) containers="$2"; shift; shift;; -d|--download-agent) download=1; shift;; -s|--share-configs) share_configs=1; shift;; -n|--network) addr="$2"; shift; shift;; -r|--restore-libvirt) restore=1; shift;; -p|--restore-cib) restore_pcmk=1; shift;; -R|--restore-all) restore_all=1 restore=1 restore_pcmk=1 shift;; -g|--generate) generate=1; key_gen=1; shift;; -k|--key-gen) key_gen=1; shift;; -a|--add-cib) cib=1; shift;; -A|--allow-anywhere) anywhere=1; shift;; -C|--add-clone) add_clone=1; shift;; -m|--add-master) echo "-m/--add-master is deprecated (use -C/--add-clone instead)" echo add_clone=1 shift ;; -v|--verify) verify=1; shift;; "") break;; *) helptext 1;; esac done if [ $verify -eq 1 ]; then # verify virsh tool is available and that # we can connect to lxc driver. virsh -c lxc:/// list --all > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "libvirt LXC driver must be installed (could not connect 'virsh -c lxc:///')" # yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell exit 1 fi SELINUX=$(getenforce) if [ "$SELINUX" != "Enforcing" ] && [ "$SELINUX" != "Permissive" ]; then echo "SELINUX must be set to permissive or enforcing mode" exit 1 fi ps ax | grep "[l]ibvirtd" if [ $? -ne 0 ]; then echo "libvirtd must be running" exit 1 fi which rsync > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "rsync must be installed" fi which pacemaker-remoted > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "pacemaker-remoted must be installed" fi fi #strip last digits off addr addr="$(echo "$addr" | awk -F. '{print $1"."$2"."$3}')" node_exec() { ssh -o StrictHostKeyChecking=no \ -o ConnectTimeout=30 \ -o BatchMode=yes \ -l root -T "$@" } this_node() { crm_node -n } other_nodes() { crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}" } make_directory() { # argument must be full path DIR="$1" mkdir -p "$DIR" if [ $share_configs -eq 1 ]; then for node in $(other_nodes); do node_exec "$node" mkdir -p "$DIR" done fi } sync_file() { TARGET="$1" if [ $share_configs -eq 1 ]; then for node in $(other_nodes); do rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}" done fi } download_agent() { wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain chmod 755 VirtualDomain mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain } set_network() { rm -f cur_network.xml cat << END >> cur_network.xml default 41ebdb84-7134-1111-a136-91f0f1119225 END sync_file "${working_dir}"/cur_network.xml } distribute_configs() { for node in $(other_nodes); do rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*.xml "${node}:${working_dir}" rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*-filesystem "${node}:${working_dir}" done } start_network() { NODE="$1" node_exec "$NODE" <<-EOF cd "$working_dir" virsh net-info default >/dev/null 2>&1 if [ \$? -eq 0 ]; then if [ ! -f restore_default.xml ]; then virsh net-dumpxml default > restore_default.xml fi virsh net-destroy default virsh net-undefine default fi virsh net-define cur_network.xml virsh net-start default virsh net-autostart default EOF } start_network_all() { start_network "$(this_node)" if [ $share_configs -eq 1 ]; then for node in $(other_nodes); do start_network "$node" done fi } add_hosts_entry() { IP="$1" HNAME="$2" echo "$IP $HNAME" >>/etc/hosts if [ $share_configs -eq 1 ]; then for node in $(other_nodes); do node_exec "$node" "echo $IP $HNAME >>/etc/hosts" done fi } generate_key() { if [ ! -e /etc/pacemaker/authkey ]; then make_directory /etc/pacemaker dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1 sync_file /etc/pacemaker/authkey fi } generate() { set_network # Generate libvirt domains in xml for (( c=1; c <= containers; c++ )) do # Clean any previous definition rm -rf "lxc$c.xml" "lxc$c-filesystem" # Create a basic filesystem with run directories for dir in $run_dirs; do mkdir -p "lxc$c-filesystem/$dir" done # Create libvirt definition suffix=$((10 + c)) prefix="$(echo "$addr" | awk -F. '{print $1"."$2}')" subnet="$(echo "$addr" | awk -F. '{print $3}')" while [ $suffix -gt 255 ]; do subnet=$((subnet + 1)) suffix=$((subnet - 255)) done cip="$prefix.$subnet.$suffix" cat << END >> lxc$c.xml lxc$c 200704 exe $working_dir/lxc$c-filesystem/launch-helper END for dir in $run_dirs; do cat << END >> lxc$c.xml END done cat << END >> lxc$c.xml END # Create CIB definition rm -f "container$c.cib" cat << END >> "container$c.cib" END # Create container init rm -f "lxc$c-filesystem/launch-helper" cat << END >> "lxc$c-filesystem/launch-helper" #!@BASH_PATH@ ip -f inet addr add "$cip/24" dev eth0 ip link set eth0 up ip route add default via "$addr.1" hostname "lxc$c" df > "$working_dir/lxc$c-filesystem/disk_usage.txt" export PCMK_debugfile="@CRM_LOG_DIR@/pacemaker_remote_lxc$c.log" /usr/sbin/pacemaker-remoted END chmod 711 "lxc$c-filesystem/launch-helper" add_hosts_entry "$cip" "lxc$c" done # Create CIB fragment for a promotable clone resource cat << END > lxc-clone.cib END } +container_names() { + find . -maxdepth 1 -name "lxc*.xml" -exec basename -s .xml "{}" ";" +} + apply_cib_clone() { cibadmin -Q > cur.cib export CIB_file=cur.cib cibadmin -o resources -Mc -x lxc-clone.cib - for tmp in $(find . -maxdepth 1 -name "lxc*.xml" | sed -e 's/\.xml//g'); do + for tmp in $(container_names); do echo "" > tmp_constraint cibadmin -o constraints -Mc -x tmp_constraint done # Make sure the version changes even if the content doesn't cibadmin -B unset CIB_file cibadmin --replace -o configuration --xml-file cur.cib rm -f cur.cib } apply_cib_entries() { cibadmin -Q > cur.cib export CIB_file=cur.cib for tmp in container*.cib; do cibadmin -o resources -Mc -x "$tmp" remote_node="$(grep remote-node "${tmp}" | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p')" if [ $anywhere -eq 0 ]; then crm_resource -M -r "${tmp//\.cib/}" -H "$(this_node)" fi echo "" > tmp_constraint # Ignore any failure; this constraint is just to help with CTS when the # connectivity resources (which fail the guest nodes) are in use. cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1 for rsc in $(crm_resource -l | grep rsc_ ); do echo "" > tmp_constraint cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1 done rm -f tmp_constraint done # Make sure the version changes even if the content doesn't cibadmin -B unset CIB_file cibadmin --replace -o configuration --xml-file cur.cib rm -f cur.cib } restore_cib() { cibadmin -Q > cur.cib export CIB_file=cur.cib - for tmp in $(find . -maxdepth 1 -name "lxc*.xml" | sed -e 's/\.xml//g'); do + for tmp in $(container_names); do echo "" > tmp_constraint cibadmin -o constraints -D -x tmp_constraint echo "" > tmp_constraint cibadmin -o constraints -D -x tmp_constraint for rsc in $(crm_resource -l | grep rsc_ ); do echo "" > tmp_constraint cibadmin -o constraints -D -x tmp_constraint done rm -f tmp_constraint done cibadmin -o resources -D -x lxc-clone.cib for tmp in container*.cib; do tmp="${tmp//\.cib/}" crm_resource -U -r "$tmp" -H "$(this_node)" crm_resource -D -r "$tmp" -t primitive done # Make sure the version changes even if the content doesn't cibadmin -B unset CIB_file cibadmin --replace -o configuration --xml-file cur.cib rm -f cur.cib # Allow the cluster to stabilize before continuing crm_resource --wait # Purge nodes from caches and CIB status section - for tmp in $(find . -maxdepth 1 -name "lxc*.xml" | sed -e 's/\.xml//g'); do + for tmp in $(container_names); do crm_node --force --remove "$tmp" done } restore_network() { NODE="$1" node_exec "$NODE" <<-EOF cd "$working_dir" for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do virsh -c lxc:/// destroy "\$tmp" >/dev/null 2>&1 virsh -c lxc:/// undefine "\$tmp" >/dev/null 2>&1 sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts done virsh net-destroy default >/dev/null 2>&1 virsh net-undefine default >/dev/null 2>&1 if [ -f restore_default.xml ]; then virsh net-define restore_default.xml virsh net-start default rm restore_default.xml fi EOF echo "Containers destroyed and default network restored on $NODE" } restore_libvirt() { restore_network "$(this_node)" if [ $share_configs -eq 1 ]; then for node in $(other_nodes); do restore_network "$node" done fi } restore_files() { find . -maxdepth 1 -not -name "lxc*.xml" -a -not -name . -exec rm -rf "{}" ";" if [ $share_configs -eq 1 ]; then for node in $(other_nodes); do node_exec "$node" rm -rf \ "$working_dir"/lxc*-filesystem \ "$working_dir"/cur_network.xml done fi } make_directory "$working_dir" cd "$working_dir" || exit 1 # Generate files as requested if [ $download -eq 1 ]; then download_agent fi if [ $key_gen -eq 1 ]; then generate_key fi if [ $generate -eq 1 ]; then generate fi if [ $share_configs -eq 1 ]; then distribute_configs fi if [ $generate -eq 1 ]; then start_network_all fi # Update cluster as requested if [ $cib -eq 1 ]; then apply_cib_entries fi if [ $add_clone -eq 1 ]; then apply_cib_clone fi # Restore original state as requested if [ $restore_pcmk -eq 1 ]; then restore_cib fi if [ $restore -eq 1 ]; then restore_libvirt fi if [ $restore_all -eq 1 ]; then restore_files fi # vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80: