diff --git a/cts/lxc_autogen.sh.in b/cts/lxc_autogen.sh.in
index 7071731bd8..c8f977f114 100644
--- a/cts/lxc_autogen.sh.in
+++ b/cts/lxc_autogen.sh.in
@@ -1,531 +1,535 @@
#!@BASH_PATH@
#
# Copyright 2013-2020 the Pacemaker project contributors
#
# The version control history for this file may have further details.
#
# This source code is licensed under the GNU General Public License version 2
# or later (GPLv2+) WITHOUT ANY WARRANTY.
#
containers="2"
download=0
share_configs=0
# different than default libvirt network in case this is run nested in a KVM instance
addr="192.168.123.1"
restore=0
restore_pcmk=0
restore_all=0
generate=0
key_gen=0
cib=0
anywhere=0
add_clone=0
verify=0
working_dir="@CRM_CONFIG_CTS@/lxc"
run_dirs="/run /var/run /usr/var/run"
# must be on one line b/c used inside quotes
SSH_RSYNC_OPTS="-o UserKnownHostsFile=/dev/null -o BatchMode=yes -o StrictHostKeyChecking=no"
function helptext() {
echo "lxc_autogen.sh - generate libvirt LXC containers for testing purposes"
echo ""
echo "Usage: lxc-autogen [options]"
echo ""
echo "Options:"
echo "-g, --generate Generate libvirt LXC environment in directory this script is run from"
echo "-k, --key-gen Generate Pacemaker Remote key only"
echo "-r, --restore-libvirt Restore the default network and libvirt config to before this script ran"
echo "-p, --restore-cib Remove CIB entries this script generated"
echo "-R, --restore-all Restore both libvirt and CIB, and clean working directory"
echo " (libvirt xml files are not removed, so resource can be stopped properly)"
echo ""
echo "-A, --allow-anywhere Allow the containers to live anywhere in the cluster"
echo "-a, --add-cib Add CIB entries to create a guest node for each LXC instance"
echo "-C, --add-clone Add promotable clone resource shared between LXC guest nodes"
echo "-d, --download-agent Download and install latest VirtualDomain agent"
echo "-s, --share-configs Synchronize on all known cluster nodes"
echo "-c, --containers Specify number of containers to generate (default $containers; used with -g)"
echo "-n, --network Network to override libvirt default (example: -n 192.168.123.1; used with -g)"
echo "-v, --verify Verify environment is capable of running LXC"
echo ""
exit "$1"
}
while true ; do
case "$1" in
--help|-h|-\?) helptext 0;;
-c|--containers) containers="$2"; shift; shift;;
-d|--download-agent) download=1; shift;;
-s|--share-configs) share_configs=1; shift;;
-n|--network) addr="$2"; shift; shift;;
-r|--restore-libvirt) restore=1; shift;;
-p|--restore-cib) restore_pcmk=1; shift;;
-R|--restore-all)
restore_all=1
restore=1
restore_pcmk=1
shift;;
-g|--generate) generate=1; key_gen=1; shift;;
-k|--key-gen) key_gen=1; shift;;
-a|--add-cib) cib=1; shift;;
-A|--allow-anywhere) anywhere=1; shift;;
-C|--add-clone|-m|--add-master) add_clone=1; shift;;
-v|--verify) verify=1; shift;;
"") break;;
*) helptext 1;;
esac
done
if [ $verify -eq 1 ]; then
# verify virsh tool is available and that
# we can connect to lxc driver.
virsh -c lxc:/// list --all > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "libvirt LXC driver must be installed (could not connect 'virsh -c lxc:///')"
# yum install -y libvirt-daemon-driver-lxc libvirt-daemon-lxc libvirt-login-shell
exit 1
fi
SELINUX=$(getenforce)
if [ "$SELINUX" != "Enforcing" ] && [ "$SELINUX" != "Permissive" ]; then
echo "SELINUX must be set to permissive or enforcing mode"
exit 1
fi
ps ax | grep "[l]ibvirtd"
if [ $? -ne 0 ]; then
echo "libvirtd must be running"
exit 1
fi
which rsync > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "rsync must be installed"
fi
which pacemaker-remoted > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "pacemaker-remoted must be installed"
fi
fi
#strip last digits off addr
addr="$(echo "$addr" | awk -F. '{print $1"."$2"."$3}')"
node_exec() {
ssh -o StrictHostKeyChecking=no \
-o ConnectTimeout=30 \
-o BatchMode=yes \
-l root -T "$@"
}
this_node()
{
crm_node -n
}
other_nodes()
{
crm_node -l | awk "\$2 != \"$(this_node)\" {print \$2}"
}
make_directory()
{
# argument must be full path
DIR="$1"
mkdir -p "$DIR"
if [ $share_configs -eq 1 ]; then
for node in $(other_nodes); do
node_exec "$node" mkdir -p "$DIR"
done
fi
}
sync_file()
{
TARGET="$1"
if [ $share_configs -eq 1 ]; then
for node in $(other_nodes); do
rsync -ave "ssh $SSH_RSYNC_OPTS" "$TARGET" "${node}:${TARGET}"
done
fi
}
download_agent()
{
wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain
chmod 755 VirtualDomain
mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
sync_file /usr/lib/ocf/resource.d/heartbeat/VirtualDomain
}
set_network()
{
rm -f cur_network.xml
cat << END >> cur_network.xml
default41ebdb84-7134-1111-a136-91f0f1119225
END
sync_file "${working_dir}"/cur_network.xml
}
distribute_configs()
{
for node in $(other_nodes); do
rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*.xml "${node}:${working_dir}"
rsync -ave "ssh $SSH_RSYNC_OPTS" "${working_dir}"/lxc*-filesystem "${node}:${working_dir}"
done
}
start_network()
{
NODE="$1"
node_exec "$NODE" <<-EOF
cd "$working_dir"
virsh net-info default >/dev/null 2>&1
if [ \$? -eq 0 ]; then
if [ ! -f restore_default.xml ]; then
virsh net-dumpxml default > restore_default.xml
fi
virsh net-destroy default
virsh net-undefine default
fi
virsh net-define cur_network.xml
virsh net-start default
virsh net-autostart default
EOF
}
start_network_all()
{
start_network "$(this_node)"
if [ $share_configs -eq 1 ]; then
for node in $(other_nodes); do
start_network "$node"
done
fi
}
add_hosts_entry()
{
IP="$1"
HNAME="$2"
echo "$IP $HNAME" >>/etc/hosts
if [ $share_configs -eq 1 ]; then
for node in $(other_nodes); do
node_exec "$node" "echo $IP $HNAME >>/etc/hosts"
done
fi
}
generate_key()
{
if [ ! -e /etc/pacemaker/authkey ]; then
make_directory /etc/pacemaker
dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1
sync_file /etc/pacemaker/authkey
fi
}
generate()
{
set_network
# Generate libvirt domains in xml
for (( c=1; c <= containers; c++ ))
do
# Clean any previous definition
rm -rf "lxc$c.xml" "lxc$c-filesystem"
# Create a basic filesystem with run directories
for dir in $run_dirs; do
mkdir -p "lxc$c-filesystem/$dir"
done
# Create libvirt definition
suffix=$((10 + c))
prefix="$(echo "$addr" | awk -F. '{print $1"."$2}')"
subnet="$(echo "$addr" | awk -F. '{print $3}')"
while [ $suffix -gt 255 ]; do
subnet=$((subnet + 1))
suffix=$((subnet - 255))
done
cip="$prefix.$subnet.$suffix"
cat << END >> lxc$c.xml
lxc$c200704exe$working_dir/lxc$c-filesystem/launch-helper
END
for dir in $run_dirs; do
cat << END >> lxc$c.xml
END
done
cat << END >> lxc$c.xml
END
# Create CIB definition
rm -f "container$c.cib"
cat << END >> "container$c.cib"
+
+
+
END
# Create container init
rm -f "lxc$c-filesystem/launch-helper"
cat << END >> "lxc$c-filesystem/launch-helper"
#!@BASH_PATH@
ip -f inet addr add "$cip/24" dev eth0
ip link set eth0 up
ip route add default via "$addr.1"
hostname "lxc$c"
df > "$working_dir/lxc$c-filesystem/disk_usage.txt"
export PCMK_debugfile="@CRM_LOG_DIR@/pacemaker_remote_lxc$c.log"
/usr/sbin/pacemaker-remoted
END
chmod 711 "lxc$c-filesystem/launch-helper"
add_hosts_entry "$cip" "lxc$c"
done
# Create CIB fragment for a promotable clone resource
cat << END > lxc-clone.cib
-
+
+
END
}
apply_cib_clone()
{
cibadmin -Q > cur.cib
export CIB_file=cur.cib
cibadmin -o resources -Mc -x lxc-clone.cib
for tmp in $(find . -maxdepth 1 -name "lxc*.xml" | sed -e 's/\.xml//g'); do
echo "" > tmp_constraint
cibadmin -o constraints -Mc -x tmp_constraint
done
# Make sure the version changes even if the content doesn't
cibadmin -B
unset CIB_file
cibadmin --replace -o configuration --xml-file cur.cib
rm -f cur.cib
}
apply_cib_entries()
{
cibadmin -Q > cur.cib
export CIB_file=cur.cib
for tmp in container*.cib; do
cibadmin -o resources -Mc -x "$tmp"
remote_node="$(grep remote-node "${tmp}" | sed -n -e 's/^.*value=\"\(.*\)\".*/\1/p')"
if [ $anywhere -eq 0 ]; then
crm_resource -M -r "${tmp//\.cib/}" -H "$(this_node)"
fi
echo "" > tmp_constraint
# Ignore any failure; this constraint is just to help with CTS when the
# connectivity resources (which fail the guest nodes) are in use.
cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
for rsc in $(crm_resource -l | grep rsc_ ); do
echo "" > tmp_constraint
cibadmin -o constraints -Mc -x tmp_constraint > /dev/null 2>&1
done
rm -f tmp_constraint
done
# Make sure the version changes even if the content doesn't
cibadmin -B
unset CIB_file
cibadmin --replace -o configuration --xml-file cur.cib
rm -f cur.cib
}
restore_cib()
{
cibadmin -Q > cur.cib
export CIB_file=cur.cib
for tmp in $(find . -maxdepth 1 -name "lxc*.xml" | sed -e 's/\.xml//g'); do
echo "" > tmp_constraint
cibadmin -o constraints -D -x tmp_constraint
echo "" > tmp_constraint
cibadmin -o constraints -D -x tmp_constraint
for rsc in $(crm_resource -l | grep rsc_ ); do
echo "" > tmp_constraint
cibadmin -o constraints -D -x tmp_constraint
done
rm -f tmp_constraint
done
cibadmin -o resources -D -x lxc-clone.cib
for tmp in container*.cib; do
tmp="${tmp//\.cib/}"
crm_resource -U -r "$tmp" -H "$(this_node)"
crm_resource -D -r "$tmp" -t primitive
done
# Make sure the version changes even if the content doesn't
cibadmin -B
unset CIB_file
cibadmin --replace -o configuration --xml-file cur.cib
rm -f cur.cib
# Allow the cluster to stabilize before continuing
crm_resource --wait
# Purge nodes from caches and CIB status section
for tmp in $(find . -maxdepth 1 -name "lxc*.xml" | sed -e 's/\.xml//g'); do
crm_node --force --remove "$tmp"
done
}
restore_network()
{
NODE="$1"
node_exec "$NODE" <<-EOF
cd "$working_dir"
for tmp in \$(ls lxc*.xml | sed -e 's/\.xml//g'); do
virsh -c lxc:/// destroy "\$tmp" >/dev/null 2>&1
virsh -c lxc:/// undefine "\$tmp" >/dev/null 2>&1
sed -i.bak "/...\....\....\..* \${tmp}/d" /etc/hosts
done
virsh net-destroy default >/dev/null 2>&1
virsh net-undefine default >/dev/null 2>&1
if [ -f restore_default.xml ]; then
virsh net-define restore_default.xml
virsh net-start default
rm restore_default.xml
fi
EOF
echo "Containers destroyed and default network restored on $NODE"
}
restore_libvirt()
{
restore_network "$(this_node)"
if [ $share_configs -eq 1 ]; then
for node in $(other_nodes); do
restore_network "$node"
done
fi
}
restore_files()
{
find . -maxdepth 1 -not -name "lxc*.xml" -a -not -name . -exec rm -rf "{}" ";"
if [ $share_configs -eq 1 ]; then
for node in $(other_nodes); do
node_exec "$node" rm -rf \
"$working_dir"/lxc*-filesystem \
"$working_dir"/cur_network.xml
done
fi
}
make_directory "$working_dir"
cd "$working_dir" || exit 1
# Generate files as requested
if [ $download -eq 1 ]; then
download_agent
fi
if [ $key_gen -eq 1 ]; then
generate_key
fi
if [ $generate -eq 1 ]; then
generate
fi
if [ $share_configs -eq 1 ]; then
distribute_configs
fi
if [ $generate -eq 1 ]; then
start_network_all
fi
# Update cluster as requested
if [ $cib -eq 1 ]; then
apply_cib_entries
fi
if [ $add_clone -eq 1 ]; then
apply_cib_clone
fi
# Restore original state as requested
if [ $restore_pcmk -eq 1 ]; then
restore_cib
fi
if [ $restore -eq 1 ]; then
restore_libvirt
fi
if [ $restore_all -eq 1 ]; then
restore_files
fi
# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80: