diff --git a/.gitignore b/.gitignore index d649215e6e..a2db24306b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,214 +1,216 @@ # Common \#* .\#* GPATH GRTAGS GTAGS TAGS Makefile Makefile.in .deps .dirstamp .libs *.pc *.pyc *.bz2 *.tar.gz *.rpm *.la *.lo *.o *~ *.gcda *.gcno # Autobuild aclocal.m4 autoconf autoheader autom4te.cache/ automake build.counter compile config.guess config.log config.status config.sub configure depcomp install-sh include/stamp-* libtool libtool.m4 ltdl.m4 libltdl ltmain.sh missing py-compile /m4/argz.m4 /m4/ltargz.m4 /m4/ltoptions.m4 /m4/ltsugar.m4 /m4/ltversion.m4 /m4/lt~obsolete.m4 test-driver ylwrap # Configure targets /cts/CTS.py /cts/CTSlab.py /cts/CTSvars.py /cts/LSBDummy /cts/OCFIPraTest.py /cts/benchmark/clubench /cts/cluster_test /cts/cts /cts/cts-cli /cts/cts-coverage /cts/cts-exec /cts/cts-fencing /cts/cts-log-watcher /cts/cts-regression /cts/cts-scheduler /cts/cts-support /cts/fence_dummy /cts/lxc_autogen.sh /cts/pacemaker-cts-dummyd /cts/pacemaker-cts-dummyd@.service /daemons/execd/pacemaker_remote /daemons/execd/pacemaker_remote.service /daemons/fenced/fence_legacy /daemons/pacemakerd/pacemaker /daemons/pacemakerd/pacemaker.combined.upstart /daemons/pacemakerd/pacemaker.service /daemons/pacemakerd/pacemaker.upstart /doc/Doxyfile /extra/logrotate/pacemaker /extra/resources/ClusterMon /extra/resources/HealthSMART /extra/resources/SysInfo /extra/resources/ifspeed /extra/resources/o2cb include/config.h include/config.h.in include/crm_config.h publican.cfg /tools/cibsecret /tools/crm_error /tools/crm_failcount /tools/crm_master /tools/crm_mon.service /tools/crm_mon.upstart /tools/crm_report /tools/crm_rule /tools/crm_standby /tools/report.collector /tools/report.common # Build targets *.7 *.7.xml *.7.html *.8 *.8.xml *.8.html /daemons/attrd/pacemaker-attrd /daemons/based/pacemaker-based /daemons/based/cibmon /daemons/controld/pacemaker-controld /daemons/execd/cts-exec-helper /daemons/execd/pacemaker-execd /daemons/execd/pacemaker-remoted /daemons/fenced/cts-fence-helper /daemons/fenced/pacemaker-fenced /daemons/fenced/pacemaker-fenced.xml /daemons/pacemakerd/pacemakerd /daemons/schedulerd/pacemaker-schedulerd /daemons/schedulerd/pacemaker-schedulerd.xml /doc/*/tmp/** /doc/*/publish /doc/*.build /doc/*/en-US/Ap-*.xml /doc/*/en-US/Ch-*.xml /doc/.ABI-build /doc/HTML /doc/abi_dumps /doc/abi-check /doc/acls.html /doc/api/* /doc/compat_reports /doc/crm_fencing.html /doc/publican-catalog* /doc/shared/en-US/*.xml /doc/shared/en-US/images/pcmk-*.png /doc/shared/en-US/images/Policy-Engine-*.png /maint/testcc scratch /tools/attrd_updater /tools/cibadmin /tools/crmadmin /tools/crm_attribute /tools/crm_diff /tools/crm_mon /tools/crm_node /tools/crm_resource /tools/crm_shadow /tools/crm_simulate /tools/crm_ticket /tools/crm_verify /tools/iso8601 /tools/stonith_admin xml/crm.dtd xml/pacemaker*.rng xml/versions.rng lib/gnu/libgnu.a lib/gnu/stdalign.h *.coverity # Test detritus /cts/.regression.failed.diff /cts/scheduler/*.ref /cts/scheduler/*.up /cts/scheduler/*.up.err /cts/scheduler/bug-rh-1097457.log /cts/scheduler/bug-rh-1097457.trs /cts/scheduler/shadow.* /cts/test-suite.log /xml/test-*/*.up /xml/test-*/*.up.err +/xml/assets/*.rng /xml/assets/diffview.js +/xml/assets/xmlcatalog # Release maintenance detritus /maint/gnulib # Formerly built files (helps when jumping back and forth in checkout) /.ABI-build /Doxyfile /HTML /abi_dumps /abi-check /compat_reports /attrd /cib /coverage.sh /crmd /cts/HBDummy /doc/Clusters_from_Scratch.txt /doc/Pacemaker_Explained.txt /fencing /lrmd /mcp /pengine #Other mock pacemaker*.spec coverity-* logs *.patch *.diff *.sed *.orig *.rej *.swp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ddac78ead..f3fb327e9c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,274 +1,274 @@ -# Copyright 2018 Red Hat, Inc. -# Author: Jan Pokorny -# Part of pacemaker project +# Copyright 2018-2019 the Pacemaker project contributors +# +# The version control history for this file may have further details. +# # SPDX-License-Identifier: FSFAP # Conventions: # 0/ subject to change: compatibility with GitLab (CE) as old as 10.8 # 1/ heavy reuse (compatible map merges for now, see fedora-rawhide example) # 2/ consolidated prefixes for custom variable names: GLOBAL_* and LOCAL_* variables: GIT_DEPTH: "16" # nice-to-have automatic "bisect" for even more convenience stages: - buildenv - build - test # currently, 1:1 with the eponymous stage .buildenv: &buildenv-anchor stage: buildenv # currently, subset of all jobs pertaining "build" stage .compilation: &compilation-anchor variables: GLOBAL_COMPILATION: "1" # bogus content so that this anchor is not empty # only since GitLab 11.4 # changes: # - "configure.ac" # - "**/*.[ch]" # - "**/*.xml" # - "**/*.xsl" # - "**/Makefile.am" # - ".gitlab-ci.yml" # # Fedora Rawhide # # In case of failure imposed with branching/changed GPG keys, downgrade, e.g.: # sed -i.orig 's|fedora-rawhide|fedora|g;s|rawhide|fedora|g;s|:rawhide|:29|' \ # .gitlab-ci.yml # and run it through your side branch (omit such changes for upstream request). # .fedora-rawhide: &fedora-rawhide-anchor tags: - docker image: registry.fedoraproject.org/fedora:rawhide # down? mere fedora:rawhide .fedora-rawhide-variables-rpm: &fedora-rawhide-variables-rpm-anchor # below are captured major sets of packages that need to be installed on top # of what the base image contains; some packages named individually later; # transitive dependencies preferably kept at minimum, i.e., w/o proliferation GLOBAL_RPM_BR_BASIC: autoconf automake pkgconf-pkg-config libtool libtool-ltdl-devel make # with these two, first word denotes also the derived value of CC variable GLOBAL_RPM_BR_GCC: gcc GLOBAL_RPM_BR_CLANG: clang # compiler-rt # for when with coverage GLOBAL_RPM_BR_LIBS_PLAIN: libuuid-devel libxslt-devel bzip2-devel # note a special trick using \v escaped character, since it's not in IFS # (word separators) in Bash, while fulfilling the requirements in libdnf # (cf. https://github.com/rpm-software-management/libdnf/pull/476) GLOBAL_RPM_BR_LIBS_PKGCONFIG: "pkgconfig(glib-2.0)\v>=2.16.0 pkgconfig(libxml-2.0) pkgconfig(libqb)\v>=0.13 pkgconfig(libcpg)" GLOBAL_RPM_REQ_LIBS: bzip2-libs glib2 # not dragged transitively .fedora-rawhide-gcc: &fedora-rawhide-gcc-anchor <<: *fedora-rawhide-anchor #extends: .fedora-rawhide variables: <<: *fedora-rawhide-variables-rpm-anchor LOCAL_RPM_COMPILER: $GLOBAL_RPM_BR_GCC LOCAL_WITH_COVERAGE: --with-coverage .fedora-rawhide-clang: &fedora-rawhide-clang-anchor <<: *fedora-rawhide-anchor #extends: .fedora-rawhide variables: <<: *fedora-rawhide-variables-rpm-anchor LOCAL_RPM_COMPILER: $GLOBAL_RPM_BR_CLANG LOCAL_WITH_COVERAGE: "" # override from enabling in GCC case for now # buildenv buildenv:fedora-rawhide:info: # to gather some info about the image itself/container settings <<: *buildenv-anchor #extends: .buildenv <<: *fedora-rawhide-anchor #extends: .fedora-rawhide variables: GIT_STRATEGY: "none" # invariant stage, not project dependence at all script: - cat /proc/meminfo | grep Free - df -H - lsblk - nproc - ulimit -a - uname -r - dnf --version buildenv:fedora-rawhide:rpms: # primarily to download RPMs, but alas, "download" plugin not baked into image <<: *buildenv-anchor #extends: .buildenv <<: *fedora-rawhide-anchor #extends: .fedora-rawhide variables: <<: *fedora-rawhide-variables-rpm-anchor GIT_STRATEGY: "none" # invariant stage, not project dependence at all cache: key: "fedora-rawhide-dnf" paths: - ".cache/fedora-rawhide/dnf/rawhide-*" - - "xml/.relaxng.org/relaxng.rng" + - "xml/assets/*.rng" script: - - mkdir -p .cache/fedora-rawhide/dnf xml/.relaxng.org + - mkdir -p .cache/fedora-rawhide/dnf - pushd .cache/fedora-rawhide - mv dnf/rawhide-* /var/cache/dnf 2>/dev/null || true - dnf-custom() { timeout 192 dnf -y --noplugins --nodocs --setopt=tsflags=test --setopt=install_weak_deps=0 --setopt=keepcache=1 --setopt=autocheck_running_kernel=0 --disablerepo='*' --enablerepo=rawhide "$@"; }; dnf-custom install $GLOBAL_RPM_BR_BASIC $GLOBAL_RPM_BR_GCC $GLOBAL_RPM_BR_CLANG $GLOBAL_RPM_BR_LIBS_PLAIN $GLOBAL_RPM_BR_LIBS_PKGCONFIG $GLOBAL_RPM_REQ_LIBS findutils libxml libxslt psmisc which gcovr - mv /var/cache/dnf/rawhide-* dnf - popd # we don't have any other warm-up-cache job around, so piggyback it here - - test -s xml/.relaxng.org/relaxng.rng 2>/dev/null - || curl -SsLo xml/.relaxng.org/relaxng.rng - 'http://relaxng.org/relaxng.rng' + - test -s xml/assets/relaxng.rng 2>/dev/null + || curl -SsLo xml/assets/relaxng.rng + 'https://raw.githubusercontent.com/relaxng/relaxng.org/master/relaxng.rng' + - test -s xml/assets/xslt.rng 2>/dev/null + || curl -SsLo xml/assets/xslt.rng + 'https://raw.githubusercontent.com/relaxng/jing-trang/master/eg/xslt.rng' # build maint:fedora-rawhide: # to run some compilation-less checks <<: *fedora-rawhide-anchor #extends: .fedora-rawhide stage: build cache: key: "fedora-rawhide-dnf" policy: pull paths: - ".cache/fedora-rawhide/dnf/rawhide-*" - - "xml/.relaxng.org/relaxng.rng" + - "xml/assets/*.rng" before_script: - mkdir -p .cache/fedora-rawhide/dnf - pushd .cache/fedora-rawhide - mv dnf/rawhide-* /var/cache/dnf 2>/dev/null || true - dnf-custom() { timeout 96 dnf -y --cacheonly --noplugins --nodocs --setopt=install_weak_deps=0 --setopt=autocheck_running_kernel=0 --disablerepo='*' --enablerepo=rawhide "$@"; }; dnf-custom install findutils make libxml libxslt - popd - ( cd xml; { cat Makefile.am; printf 'hack_rng\x3a %s' '${RNG_generated}'; } | make -f- top_srcdir=$(pwd)/.. top_builddir=$(pwd)/.. hack_rng ) script: - echo 'looking for presence of control characters...'; { git ls-files | xargs grep -Ensv "^([^[:cntrl:]]*|$(printf '\t'))*$"||:; } 2>/dev/null | { ! grep -Ev '^Binary file' && echo 'ALL OK'; }; - # ./regression.sh && ./regression.sh -B && ./regression.sh -S && { - cd xml; - ./regression.sh && ./regression.sh -B && { + ./regression.sh && ./regression.sh -B && ./regression.sh -S && { schemas=; for schema in *.rng; do case ${schema} in *cibtr*);; *)schemas="${schemas} ${schema}";; esac; done; - test -s .relaxng.org/relaxng.rng 2>/dev/null - || curl --create-dirs -Lo .relaxng.org/relaxng.rng - 'http://relaxng.org/relaxng.rng'; - xmllint --noout --relaxng .relaxng.org/relaxng.rng ${schemas}; + xmllint --noout --relaxng assets/relaxng.rng ${schemas}; } build:fedora-rawhide:gcc: &build-fedora-rawhide-gcc-anchor # to build using GCC <<: *compilation-anchor #extends: .compilation <<: *fedora-rawhide-gcc-anchor #extends: .fedora-rawhide-gcc stage: build cache: key: "fedora-rawhide-dnf" policy: pull paths: - ".cache/fedora-rawhide/dnf/rawhide-*" artifacts: name: "fedora-rawhide-gcc-$CI_COMMIT_REF_SLUG" untracked: true expire_in: "10h" before_script: - mkdir -p .cache/fedora-rawhide/dnf - pushd .cache/fedora-rawhide - mv dnf/rawhide-* /var/cache/dnf 2>/dev/null || true - dnf-custom() { timeout 96 dnf -y --cacheonly --noplugins --nodocs --setopt=install_weak_deps=0 --setopt=autocheck_running_kernel=0 --disablerepo='*' --enablerepo=rawhide "$@"; }; dnf-custom install $GLOBAL_RPM_BR_BASIC $LOCAL_RPM_COMPILER $GLOBAL_RPM_BR_LIBS_PLAIN $GLOBAL_RPM_BR_LIBS_PKGCONFIG - popd script: - ./autogen.sh - CC=$(echo $LOCAL_RPM_COMPILER | cut -d ' ' -f1) ./configure --enable-silent-rules $LOCAL_WITH_COVERAGE || { cat config.log; false; } - timeout 480 make -j$(nproc) # -j$(($(nproc) + 1)) all - find \( -name '*.l[ao]' -o -name 'config.*' -o -name configure -o -name Makefile.in -o -name Makefile \) -delete # *.o need to remain for test build:fedora-rawhide:clang: # to build using Clang <<: *build-fedora-rawhide-gcc-anchor #extends build:fedora-rawhide:gcc <<: *fedora-rawhide-clang-anchor #extends: .fedora-rawhide-clang # test test:fedora-rawhide:gcc: &test-fedora-rawhide-gcc-anchor # to test the result of GCC build + measure coverage for that <<: *compilation-anchor #extends: .compilation <<: *fedora-rawhide-gcc-anchor #extends: .fedora-rawhide-gcc stage: test cache: key: "fedora-rawhide-dnf" policy: pull paths: - ".cache/fedora-rawhide/dnf/rawhide-*" dependencies: - build:fedora-rawhide:gcc before_script: - mkdir -p .cache/fedora-rawhide/dnf - pushd .cache/fedora-rawhide - mv dnf/rawhide-* /var/cache/dnf 2>/dev/null || true - dnf-custom() { timeout 96 dnf -y --cacheonly --noplugins --nodocs --setopt=install_weak_deps=0 --setopt=autocheck_running_kernel=0 --disablerepo='*' --enablerepo=rawhide "$@"; }; dnf-custom install $LOCAL_RPM_COMPILER $GLOBAL_RPM_BR_LIBS_PLAIN $GLOBAL_RPM_BR_LIBS_PKGCONFIG psmisc which $(test -z "$LOCAL_WITH_COVERAGE" || echo gcovr) findutils - popd script: - timeout 600 ./cts/cts-regression -V cli scheduler after_script: - gcovr -r . - find -name 'core*' || true coverage: '/^TOTAL.*\s+(\d+\%)$/' test:fedora-rawhide:clang: # to test the result of Clang build (+ possibly measure coverage for that) <<: *test-fedora-rawhide-gcc-anchor #extends: test:fedora-rawhide:gcc <<: *fedora-rawhide-clang-anchor #extends: .fedora-rawhide-clang dependencies: - build:fedora-rawhide:clang diff --git a/.travis.yml b/.travis.yml index c2b9854a09..f382e22e7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,135 +1,139 @@ +# Copyright 2012-2019 the Pacemaker project contributors +# +# The version control history for this file may have further details. + # Control file for the Travis autobuilder # https://docs.travis-ci.com/user/customizing-the-build/ language: c matrix: include: - compiler: gcc env: MAINT_EXTRA=0 - compiler: clang env: MAINT_EXTRA=0 #- compiler: cov-build # env: MAINT_EXTRA=0 - compiler: gcc env: MAINT_EXTRA=1 env: global: # -- BEGIN Coverity Scan ENV # Used by https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh # The build command with all of the arguments that you would apply to a manual `cov-build` - COVERITY_SCAN_BUILD_COMMAND="make" # Email address for notifications related to this build - OWNER_EMAIL="andrew@beekhof.net" # Regular expression selects on which branches to run analysis # Be aware of quotas. Do not run on every branch/commit - COVERITY_SCAN_BRANCH_PATTERN="1.1" # COVERITY_SCAN_TOKEN via "travis encrypt" using the repo's public key - secure: "qnrF7L8RejLUY7URdNe7XP4Hu4R55C0tvAuMRg4EjVtelOpw+nIgA7BLiX19q/70VjFuKcGnMhW28TdYl0uwMMdWKKxmwTim04Sy3UfOE2BPeuQOBphr+8s9gd0U1MO8j2dZ84A40t5Mkk946wWZwT0okpjOr/PfBOZkU3o87FM=" # -- END Coverity Scan ENV cache: directories: - xml/.relaxng.org # sudo add-apt-repository ppa:hotot-team before_install: - sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu/ trusty main" - sudo apt-get update -qq # To switch to Travis-CI's containerized (non-sudo) architecture, # all our dependencies need to be on Travis's whitelist: # https://github.com/travis-ci/apt-package-whitelist # # The only ones that aren't already are: # - cluster-glue-dev: see open issue: # https://github.com/travis-ci/apt-package-whitelist/issues/2936 # - resource-agents: see open issue: # https://github.com/travis-ci/apt-package-whitelist/issues/4261 # - libdbus-1-dev: see multiple open issues: # https://github.com/travis-ci/apt-package-whitelist/issues?utf8=%E2%9C%93&q=is%3Aissue+libdbus+-1-dev # (a workaround is to install libdbus-glib-1-dev, which depends on it and is whitelisted) install: - test $MAINT_EXTRA -ne 0 || sudo apt-get install -qq automake autoconf libtool python python-dev libbz2-dev libdbus-1-dev libglib2.0-dev libgnutls-dev libltdl-dev libncurses5-dev libpam0g-dev libxml2-dev libxslt1-dev uuid-dev libqb-dev libcfg-dev libcmap-dev libcorosync-common-dev libcpg-dev libquorum-dev libsam-dev libtotem-pg-dev libvotequorum-dev cluster-glue-dev resource-agents - test $MAINT_EXTRA -eq 0 || sudo apt-get install -qq make libxml2-utils xsltproc before_script: # Save and restore CC so that ./configure can pass - test $MAINT_EXTRA -ne 0 || { export CC_SAVED=$CC; export CC=$(echo ${CC} | sed s/cov-build/gcc/); ./autogen.sh; ./configure; export CC=$CC_SAVED; } - test $MAINT_EXTRA -eq 0 || ( cd xml; { cat Makefile.am; printf 'hack_rng\x3a %s' '${RNG_generated}'; } | make -f- top_srcdir=$(pwd)/.. top_builddir=$(pwd)/.. hack_rng ) script: # XXX Is the following item needed? - test $MAINT_EXTRA -ne 0 || test ${CC} = cov-build || sudo make install-exec-local || true - test $MAINT_EXTRA -ne 0 || test ${CC} = cov-build || { make && ./cts/cts-regression -V cli scheduler exec; } - test $MAINT_EXTRA -ne 0 || test ${CC} != cov-build || export CC=gcc; bash ./maint/travisci_build_coverity_scan.sh - test $MAINT_EXTRA -eq 0 || { { echo 'looking for presence of control characters...'; { git ls-files | xargs grep -Ensv "^([^[:cntrl:]]*|$(printf '\t'))*$"||:; } 2>/dev/null | { ! grep -Ev '^Binary file' && echo 'ALL OK'; }; } && ( cd xml; - ./regression.sh && ./regression.sh -B && { + ./regression.sh && ./regression.sh -B && ./regression.sh -S && { schemas=; for schema in *.rng; do case ${schema} in *cibtr*);; *)schemas="${schemas} ${schema}";; esac; done; test -s .relaxng.org/relaxng.rng 2>/dev/null - || curl --create-dirs -Lo .relaxng.org/relaxng.rng - 'http://relaxng.org/relaxng.rng'; + || curl --create-dirs -SsLo .relaxng.org/relaxng.rng + 'https://raw.githubusercontent.com/relaxng/relaxng.org/master/relaxng.rng'; xmllint --noout --relaxng .relaxng.org/relaxng.rng ${schemas}; } ); } #after_script: #after_success: after_failure: - lsb_release -a - sudo cat /etc/apt/sources.list - whoami - env | sort - cat include/config.h notifications: irc: "irc.freenode.org#pcmk" # email: # recipients: # - developers@clusterlabs.org # whitelist branches: only: - master - "1.1" - "2.0" diff --git a/xml/regression.sh b/xml/regression.sh index 75128f8e61..7478d28ea4 100755 --- a/xml/regression.sh +++ b/xml/regression.sh @@ -1,735 +1,763 @@ #!/bin/sh # Copyright 2018-2019 the Pacemaker project contributors # # The version control history for this file may have further details. # # SPDX-License-Identifier: GPL-2.0-or-later set -eu test -d assets && test -d test-2 \ || { echo 'Run me from source-tree-like location'; exit 1; } # $1=reference (can be '-' for stdin), $2=investigated # alt.: wdiff, colordiff, ... DIFF=${DIFF:-diff} DIFFOPTS=${DIFFOPTS--u} DIFFPAGER=${DIFFPAGER:-less -LRX} # $1=schema, $2=validated # alt.: jing -i RNGVALIDATOR=${RNGVALIDATOR:-xmllint --noout --relaxng} # $1=stylesheet, $2=source # alt.: Xalan, saxon, sabcmd/Sablotron (note: only validates reliably with -B) _xalan_wrapper() { { ${_XSLTPROCESSOR} "$2" "$1" 2>&1 >&3 \ | sed -e '/^Source tree node.*$/d' \ -e 's|^XSLT message: \(.*\) (Occurred.*)|\1|'; } 3>&- 3>&1 >&2 } # Sablotron doesn't translate '-' file specification to stdin # and limits the length of the output message _sabcmd_wrapper() { _sabw_sheet=${1:?} _sabw_source=${2:?} test "${_sabw_sheet}" != - || _sabw_sheet=/dev/stdin test "${_sabw_source}" != - || _sabw_source=/dev/stdin { ${_XSLTPROCESSOR} "${_sabw_sheet}" "${_sabw_source}" 2>&1 >&3 \ | sed -e '/^Warning \[code:89\]/d' \ -e 's|^ xsl:message (\(.*\))$|\1|'; } 3>&- 3>&1 >&2 } # filtered out message: https://bugzilla.redhat.com/show_bug.cgi?id=1577367 _saxon_wrapper() { { ${_XSLTPROCESSOR} "-xsl:$1" "-s:$2" -versionmsg:off 2>&1 >&3 \ | sed -e '/^Cannot find CatalogManager.properties$/d'; } 3>&- 3>&1 >&2 } XSLTPROCESSOR=${XSLTPROCESSOR:-xsltproc --nonet} _XSLTPROCESSOR=${XSLTPROCESSOR} case "${XSLTPROCESSOR}" in [Xx]alan*|*/[Xx]alan*) XSLTPROCESSOR=_xalan_wrapper;; sabcmd*|*/sabcmd*) XSLTPROCESSOR=_sabcmd_wrapper;; saxon*|*/saxon*) XSLTPROCESSOR=_saxon_wrapper;; esac HTTPPORT=${HTTPPORT:-8000} # Python's default WEBBROWSER=${WEBBROWSER:-firefox} tests= # test* names (should go first) here will become preselected default # # commons # emit_result() { _er_howmany=${1:?} # how many errors (0/anything else incl. strings) _er_subject=${2:?} _er_prefix=${3-} test -z "${_er_prefix}" || _er_prefix="${_er_prefix}: " if test "${_er_howmany}" = 0; then printf "%s%s finished OK\n" "${_er_prefix}" "${_er_subject}" else printf "%s%s encountered ${_er_howmany} errors\n" \ "${_er_prefix}" "${_er_subject}" fi } emit_error() { _ee_msg=${1:?} printf "%s\n" "${_ee_msg}" >&2 } # returns 1 + floor of base 2 logaritm for _lo0r_i in 1...255, # or 0 for _lo0r_i = 0 log2_or_0_return() { _lo0r_i=${1:?} return $(((!(_lo0r_i >> 1) && _lo0r_i) * 1 \ + (!(_lo0r_i >> 2) && _lo0r_i & (1 << 1)) * 2 \ + (!(_lo0r_i >> 3) && _lo0r_i & (1 << 2)) * 3 \ + (!(_lo0r_i >> 4) && _lo0r_i & (1 << 3)) * 4 \ + (!(_lo0r_i >> 5) && _lo0r_i & (1 << 4)) * 5 \ + (!(_lo0r_i >> 6) && _lo0r_i & (1 << 5)) * 6 \ + (!(_lo0r_i >> 7) && _lo0r_i & (1 << 6)) * 7 \ + !!(_lo0r_i >> 7) * 7 )) } # rough addition of two base 2 logarithms log2_or_0_add() { _lo0a_op1=${1:?} _lo0a_op2=${2:?} if test ${_lo0a_op1} -gt ${_lo0a_op2}; then return ${_lo0a_op1} elif test ${_lo0a_op2} -gt ${_lo0a_op1}; then return ${_lo0a_op2} elif test ${_lo0a_op1} -gt 0; then return $((_lo0a_op1 + 1)) else return ${_lo0a_op1} fi } # # test phases # # stdin: input file per line test_browser() { _tb_cleanref=0 _tb_serverpid= if ! read _tb_first; then return 1 fi cat >/dev/null 2>/dev/null # read out the rest test -f assets/diffview.js \ - || curl -Lo assets/diffview.js \ + || curl -SsLo assets/diffview.js \ 'https://raw.githubusercontent.com/prettydiff/prettydiff/2.2.8/lib/diffview.js' { which python3 >/dev/null 2>/dev/null \ && { python3 -m http.server "${HTTPPORT}" -b 127.0.0.1 \ || emit_error "Python3 HTTP server fail"; return; } \ || which python2 >/dev/null 2>/dev/null \ && { printf '%s %s\n' \ 'Python 2 backed HTTP server cannot listen at particular' \ 'address, discretion regarding firewall rules recommended!' python2 -m SimpleHTTPServer "${HTTPPORT}" \ || emit_error 'Python2 HTTP server fail'; return; } \ || emit_error 'Cannot run Python based HTTP server' ; } & _tb_serverpid=$! ${WEBBROWSER} "http://localhost:${HTTPPORT}/${_tb_first}" & printf "When finished, just press Ctrl+C or kill %d, please\n" \ "${_tb_serverpid}" wait } # -r ... whether to remove referential files as well # stdin: input file per line test_cleaner() { _tc_cleanref=0 while test $# -gt 0; do case "$1" in -r) _tc_cleanref=1;; esac shift done while read _tc_origin; do _tc_origin=${_tc_origin%.*} rm -f "${_tc_origin}.up" "${_tc_origin}.up.err" rm -f "$(dirname "${_tc_origin}")/.$(basename "${_tc_origin}").up" test ${_tc_cleanref} -eq 0 \ || rm -f "${_tc_origin}.ref" "${_tc_origin}.ref.err" done } # -a= ... action modifier to derive template name from (if any; enter/leave) # -o= ... which conventional version to deem as the transform origin test_selfcheck() { _tsc_ret=0 _tsc_action= _tsc_template= _tsc_validator= while test $# -gt 0; do case "$1" in -a=*) _tsc_action="${1#-a=}";; -o=*) _tsc_template="${1#-o=}";; esac shift done _tsc_validator="${_tsc_template:?}" _tsc_validator="cibtr-${_tsc_validator%%.*}.rng" _tsc_action=${_tsc_action:+-${_tsc_action}} _tsc_template="upgrade-${_tsc_template}${_tsc_action}.xsl" - # check schema (sub-grammar) for custom transformation mapping alone - if test -z "${_tsc_action}" \ - && ! ${RNGVALIDATOR} 'http://relaxng.org/relaxng.rng' "${_tsc_validator}"; then + # alt. https://relaxng.org/relaxng.rng + _tsc_rng_relaxng=https://raw.githubusercontent.com/relaxng/relaxng.org/master/relaxng.rng + # alt. https://github.com/ndw/xslt-relax-ng/blob/master/1.0/xslt10.rnc + _tsc_rng_xslt=https://raw.githubusercontent.com/relaxng/jing-trang/master/eg/xslt.rng + + case "${RNGVALIDATOR}" in + *xmllint*) + test -f "assets/$(basename "${_tsc_rng_relaxng}")" \ + || curl -SsLo "assets/$(basename "${_tsc_rng_relaxng}")" \ + "${_tsc_rng_relaxng}" + test -f "assets/$(basename "${_tsc_rng_xslt}")" \ + || curl -SsLo "assets/$(basename "${_tsc_rng_xslt}")" \ + "${_tsc_rng_xslt}" + test -f assets/xmlcatalog || >assets/xmlcatalog cat <<-EOF + + + + +EOF + RNGVALIDATOR=\ +"eval env \"XML_CATALOG_FILES=/etc/xml/catalog $(pwd)/assets/xmlcatalog\" +${RNGVALIDATOR}" + # not needed + #_tsc_rng_relaxng="assets/$(basename "${_tsc_rng_relaxng}")" + #_tsc_rng_xslt="assets/$(basename "${_tsc_rng_xslt}")" + ;; + esac + + # check schema (sub-grammar) for custom transformation mapping alone; + if test -z "${_tsc_action}" \ + && ! ${RNGVALIDATOR} "${_tsc_rng_relaxng}" "${_tsc_validator}"; then _tsc_ret=$((_tsc_ret + 1)) fi - # check the overall XSLT per the main grammar + said sub-grammar - if ! ${RNGVALIDATOR} \ - "$(test -f "${_tsc_validator}" \ - && echo "xslt_${_tsc_validator}" \ - || echo 'http://www.thaiopensource.com/relaxng/xslt.rng')" \ - "${_tsc_template}"; then + # check the overall XSLT per the main grammar + said sub-grammar; + test -f "${_tsc_validator}" && _tsc_validator="xslt_${_tsc_validator}" \ + || _tsc_validator="${_tsc_rng_xslt}" + if ! ${RNGVALIDATOR} "${_tsc_validator}" "${_tsc_template}"; then _tsc_ret=$((_tsc_ret + 1)) fi log2_or_0_return ${_tsc_ret} } test_explanation() { _tsc_template= while test $# -gt 0; do case "$1" in -o=*) _tsc_template="upgrade-${1#-o=}.xsl";; esac shift done ${XSLTPROCESSOR} upgrade-detail.xsl "${_tsc_template}" } # stdout: filename of the transformed file test_runner_upgrade() { _tru_template=${1:?} _tru_source=${2:?} # filename _tru_mode=${3:?} # extra modes wrt. "referential" outcome, see below _tru_ref="${_tru_source%.*}.ref" { test "$((_tru_mode & (1 << 0)))" -ne 0 \ || test -f "${_tru_ref}.err"; } \ && _tru_ref_err="${_tru_ref}.err" || _tru_ref_err=/dev/null _tru_target="${_tru_source%.*}.up" _tru_target_err="${_tru_target}.err" if test $((_tru_mode & (1 << 2))) -eq 0; then ${XSLTPROCESSOR} "${_tru_template}" "${_tru_source}" \ > "${_tru_target}" 2> "${_tru_target_err}" \ || { _tru_ref=$?; echo "${_tru_target_err}" return ${_tru_ref}; } else # when -B (deblanked outcomes handling) requested, we: # - drop blanks from the source XML # (effectively emulating pacemaker handling) # - re-drop blanks from the XSLT outcome, # which is compared with referential outcome # processed with even greedier custom deblanking # (extraneous inter-element whitespace like blank # lines will not get removed otherwise, see lower) xmllint --noblanks "${_tru_source}" \ | ${XSLTPROCESSOR} "${_tru_template}" - \ > "${_tru_target}" 2> "${_tru_target_err}" \ || { _tru_ref=$?; echo "${_tru_target_err}" return ${_tru_ref}; } # reusing variable no longer needed _tru_template="$(dirname "${_tru_target}")" _tru_template="${_tru_template}/.$(basename "${_tru_target}")" mv "${_tru_target}" "${_tru_template}" ${XSLTPROCESSOR} - "${_tru_template}" > "${_tru_target}" <<-EOF EOF fi # only respond with the flags except for "-B", i.e., when both: # - _tru_mode non-zero # - "-B" in _tru_mode is zero (hence non-zero when flipped with XOR) if test "$((_tru_mode * ((_tru_mode ^ (1 << 2)) & (1 << 2))))" -ne 0; then if test $((_tru_mode & (1 << 0))) -ne 0; then cp -a "${_tru_target}" "${_tru_ref}" cp -a "${_tru_target_err}" "${_tru_ref_err}" fi if test $((_tru_mode & (1 << 1))) -ne 0; then { "${DIFF}" ${DIFFOPTS} "${_tru_source}" "${_tru_ref}" \ && printf '\n(files match)\n'; } | ${DIFFPAGER} >&2 if test $? -ne 0; then printf "\npager failure\n" >&2 return 1 fi printf '\nIs comparison OK? ' >&2 if read _tru_answer &2; return 1;; esac else return 1 fi fi elif test -f "${_tru_ref}" && test -e "${_tru_ref_err}"; then { test "$((_tru_mode & (1 << 2)))" -eq 0 && cat "${_tru_ref}" \ || ${XSLTPROCESSOR} - "${_tru_ref}" <<-EOF EOF } \ | "${DIFF}" ${DIFFOPTS} - "${_tru_target}" >&2 \ && "${DIFF}" ${DIFFOPTS} "${_tru_ref_err}" \ "${_tru_target_err}" >&2 if test $? -ne 0; then emit_error "Outputs differ from referential ones" echo "/dev/null" return 1 fi else emit_error "Referential file(s) missing: ${_tru_ref}" echo "/dev/null" return 1 fi echo "${_tru_target}" } test_runner_validate() { _trv_schema=${1:?} _trv_target=${2:?} # filename if ! ${RNGVALIDATOR} "${_trv_schema}" "${_trv_target}" \ 2>/dev/null; then ${RNGVALIDATOR} "${_trv_schema}" "${_trv_target}" fi } # -a= ... action modifier completing template name (e.g. 2.10-(enter|leave)) # -o= ... which conventional version to deem as the transform origin # -t= ... which conventional version to deem as the transform target # -B # -D # -G ... see usage # stdin: input file per line test_runner() { _tr_mode=0 _tr_ret=0 _tr_action= _tr_schema_o= _tr_schema_t= _tr_target= _tr_template= while test $# -gt 0; do case "$1" in -a=*) _tr_action="${1#-a=}";; -o=*) _tr_template="${1#-o=}" _tr_schema_o="pacemaker-${1#-o=}.rng";; -t=*) _tr_schema_t="pacemaker-${1#-t=}.rng";; -G) _tr_mode=$((_tr_mode | (1 << 0)));; -D) _tr_mode=$((_tr_mode | (1 << 1)));; -B) _tr_mode=$((_tr_mode | (1 << 2)));; esac shift done _tr_template="upgrade-${_tr_action:-${_tr_template:?}}.xsl" if ! test -f "${_tr_schema_o:?}" || ! test -f "${_tr_schema_t:?}"; then emit_error "Origin and/or target schema missing, rerun make" return 1 fi while read _tr_origin; do printf '%-60s' "${_tr_origin}... " # pre-validate if ! test_runner_validate "${_tr_schema_o}" "${_tr_origin}"; then _tr_ret=$((_tr_ret + 1)); echo "E:pre-validate"; continue fi # upgrade if ! _tr_target=$(test_runner_upgrade "${_tr_template}" \ "${_tr_origin}" "${_tr_mode}"); then _tr_ret=$((_tr_ret + 1)); test -n "${_tr_target}" || break echo "E:upgrade" test -s "${_tr_target}" \ && { echo ---; cat "${_tr_target}" || :; echo ---; } continue fi # post-validate if ! test_runner_validate "${_tr_schema_t}" "${_tr_target}"; then _tr_ret=$((_tr_ret + 1)); echo "E:post-validate"; continue fi echo "OK" done log2_or_0_return ${_tr_ret} } # # particular test variations # -C # -S # -X # -W ... see usage # stdin: granular test specification(s) if any # test2to3() { _t23_pattern= while read _t23_spec; do _t23_spec=${_t23_spec%.xml} _t23_spec=${_t23_spec%\*} _t23_pattern="${_t23_pattern} -name ${_t23_spec}*.xml -o" done test -z "${_t23_pattern}" || _t23_pattern="( ${_t23_pattern%-o} )" find test-2 -name test-2 -o -type d -prune \ -o -name '*.xml' ${_t23_pattern} -print | env LC_ALL=C sort \ | { case " $* " in *\ -C\ *) test_cleaner;; *\ -S\ *) test_selfcheck -o=2.10;; *\ -X\ *) test_explanation -o=2.10;; *\ -W\ *) test_browser;; *) test_runner -o=2.10 -t=3.0 "$@" || return $?;; esac; } } tests="${tests} test2to3" test2to3enter() { _t23e_pattern= while read _t23e_spec; do _t23e_spec=${_t23e_spec%.xml} _t23e_spec=${_t23e_spec%\*} _t23e_pattern="${_t23e_pattern} -name ${_t23e_spec}*.xml -o" done test -z "${_t23e_pattern}" || _t23e_pattern="( ${_t23e_pattern%-o} )" find test-2-enter -name test-2-enter -o -type d -prune \ -o -name '*.xml' ${_t23e_pattern} -print | env LC_ALL=C sort \ | { case " $* " in *\ -C\ *) test_cleaner;; *\ -S\ *) test_selfcheck -a=enter -o=2.10;; *\ -W\ *) emit_result "not implemented" "option -W";; *\ -X\ *) emit_result "not implemented" "option -X";; *) test_runner -a=2.10-enter -o=2.10 -t=2.10 "$@" || return $?;; esac; } } tests="${tests} test2to3enter" test2to3leave() { _t23l_pattern= while read _t23l_spec; do _t23l_spec=${_t23l_spec%.xml} _t23l_spec=${_t23l_spec%\*} _t23l_pattern="${_t23l_pattern} -name ${_t23l_spec}*.xml -o" done test -z "${_t23l_pattern}" || _t23l_pattern="( ${_t23l_pattern%-o} )" find test-2-leave -name test-2-leave -o -type d -prune \ -o -name '*.xml' ${_t23l_pattern} -print | env LC_ALL=C sort \ | { case " $* " in *\ -C\ *) test_cleaner;; *\ -S\ *) test_selfcheck -a=leave -o=2.10;; *\ -W\ *) emit_result "not implemented" "option -W";; *\ -X\ *) emit_result "not implemented" "option -X";; *) test_runner -a=2.10-leave -o=3.0 -t=3.0 "$@" || return $?;; esac; } } tests="${tests} test2to3leave" test2to3roundtrip() { _t23rt_pattern= while read _t23tr_spec; do _t23rt_spec=${_t23rt_spec%.xml} _t23rt_spec=${_t23rt_spec%\*} _t23rt_pattern="${_t23rt_pattern} -name ${_t23rt_spec}*.xml -o" done test -z "${_t23rt_pattern}" || _t23rt_pattern="( ${_t23rt_pattern%-o} )" find test-2-roundtrip -name test-2-roundtrip -o -type d -prune \ -o -name '*.xml' ${_t23rt_pattern} -print | env LC_ALL=C sort \ | { case " $* " in *\ -C\ *) test_cleaner;; *\ -S\ *) test_selfcheck -a=roundtrip -o=2.10;; *\ -W\ *) test_browser;; *\ -X\ *) emit_result "not implemented" "option -X";; *) test_runner -a=2.10-roundtrip -o=2.10 -t=3.0 "$@" || return $?;; esac; } } tests="${tests} test2to3roundtrip" # -B # -D # -G ... see usage cts_scheduler() { _tcp_mode=0 _tcp_ret=0 _tcp_validatewith= _tcp_schema_o= _tcp_schema_t= _tcp_template= find ../cts/scheduler -name scheduler -o -type d -prune \ -o -name '*.xml' -print | env LC_ALL=C sort \ | { case " $* " in *\ -C\ *) test_cleaner -r;; *\ -S\ *) emit_result "not implemented" "option -S";; *\ -W\ *) emit_result "not implemented" "option -W";; *\ -X\ *) emit_result "not implemented" "option -X";; *) while test $# -gt 0; do case "$1" in -G) _tcp_mode=$((_tcp_mode | (1 << 0)));; -D) _tcp_mode=$((_tcp_mode | (1 << 1)));; -B) _tcp_mode=$((_tcp_mode | (1 << 2)));; esac shift done while read _tcp_origin; do _tcp_validatewith=$(${XSLTPROCESSOR} - "${_tcp_origin}" <<-EOF EOF ) _tcp_schema_t=${_tcp_validatewith} case "${_tcp_validatewith}" in 1) _tcp_schema_o=1.3;; 2) _tcp_schema_o=2.10;; # only for gradual refinement as upgrade-2.10.xsl under # active development, move to 3.x when schema v4 emerges 3) _tcp_schema_o=2.10 _tcp_schema_t=2;; *) emit_error \ "need to skip ${_tcp_origin} (schema: ${_tcp_validatewith})" continue;; esac _tcp_template="upgrade-${_tcp_schema_o}.xsl" _tcp_schema_t="pacemaker-$((_tcp_schema_t + 1)).0.rng" test "${_tcp_schema_o%%.*}" = "${_tcp_validatewith}" \ && _tcp_schema_o="pacemaker-${_tcp_schema_o}.rng" \ || _tcp_schema_o="${_tcp_schema_t}" # pre-validate if test "${_tcp_schema_o}" != "${_tcp_schema_t}" \ && ! test_runner_validate "${_tcp_schema_o}" "${_tcp_origin}"; then _tcp_ret=$((_tcp_ret + 1)); echo "E:pre-validate"; continue fi # upgrade test "$((_tcp_mode & (1 << 0)))" -ne 0 \ || ln -fs "$(pwd)/${_tcp_origin}" "${_tcp_origin%.*}.ref" if ! _tcp_target=$(test_runner_upgrade "${_tcp_template}" \ "${_tcp_origin}" "${_tcp_mode}"); then _tcp_ret=$((_tcp_ret + 1)); test -n "${_tcp_target}" || break echo "E:upgrade" test -s "${_tcp_target}" \ && { echo ---; cat "${_tcp_target}" || :; echo ---; } continue fi test "$((_tcp_mode & (1 << 0)))" -ne 0 \ || rm -f "${_tcp_origin%.*}.ref" # post-validate if ! test_runner_validate "${_tcp_schema_t}" "${_tcp_target}"; then _tcp_ret=$((_tcp_ret + 1)); echo "E:post-validate"; continue fi test "$((_tcp_mode & (1 << 0)))" -eq 0 \ || mv "${_tcp_target}" "${_tcp_origin}" done; log2_or_0_return ${_tcp_ret};; esac; } } tests="${tests} cts_scheduler" # # "framework" # # option-likes ... options to be passed down # argument-likes ... drives a test selection test_suite() { _ts_pass= _ts_select= _ts_select_full= _ts_test_specs= _ts_global_ret=0 _ts_ret=0 while test $# -gt 0; do case "$1" in -) printf '%s\n' 'waiting for tests specified at stdin...'; while read _ts_spec; do _ts_select="${_ts_spec}@$1"; done;; -*) _ts_pass="${_ts_pass} $1";; *) _ts_select_full="${_ts_select_full}@$1" _ts_select="${_ts_select}@${1%%/*}";; esac shift done _ts_select="${_ts_select}@" _ts_select_full="${_ts_select_full}@" for _ts_test in ${tests}; do _ts_test_specs= while true; do case "${_ts_select}" in *@${_ts_test}@*) _ts_test_specs="${_ts_select%%@${_ts_test}@*}"\ "@${_ts_select#*@${_ts_test}@}" if test "${_ts_test_specs}" = @; then _ts_select= # nothing left else _ts_select="${_ts_test_specs}" fi continue ;; @) case "${_ts_test}" in test*) break;; esac # filter ;; esac test -z "${_ts_test_specs}" || break continue 2 # move on to matching with next local test done _ts_test_specs= while true; do case "${_ts_select_full}" in *@${_ts_test}/*) _ts_test_full="${_ts_test}/${_ts_select_full#*@${_ts_test}/}" _ts_test_full="${_ts_test_full%%@*}" _ts_select_full="${_ts_select_full%%@${_ts_test_full}@*}"\ "@${_ts_select_full#*@${_ts_test_full}@}" _ts_test_specs="${_ts_test_specs} ${_ts_test_full#*/}" ;; *) break ;; esac done for _ts_test_spec in ${_ts_test_specs}; do printf '%s\n' "${_ts_test_spec}" done | "${_ts_test}" ${_ts_pass} || _ts_ret=$? test ${_ts_ret} = 0 \ && emit_result ${_ts_ret} "${_ts_test}" \ || emit_result "at least 2^$((_ts_ret - 1))" "${_ts_test}" log2_or_0_add ${_ts_global_ret} ${_ts_ret} _ts_global_ret=$? done if test -n "${_ts_select#@}"; then emit_error "Non-existing test(s):$(echo "${_ts_select}" \ | tr '@' ' ')" log2_or_0_add ${_ts_global_ret} 1 || _ts_global_ret=$? fi return ${_ts_global_ret} } # NOTE: big letters are dedicated for per-test-set behaviour, # small ones for generic/global behaviour usage() { printf \ '%s\n%s\n %s\n %s\n %s\n %s\n %s\n %s\n %s\n %s\n %s\n %s\n' \ "usage: $0 [-{B,C,D,G,S,X}]* \\" \ " [-|{${tests## }}*]" \ "- when no suites (arguments) provided, \"test*\" ones get used" \ "- with '-' suite specification the actual ones grabbed on stdin" \ "- use '-B' to run validate-only check suppressing blanks first" \ "- use '-C' to only cleanup ephemeral byproducts" \ "- use '-D' to review originals vs. \"referential\" outcomes" \ "- use '-G' to generate \"referential\" outcomes" \ "- use '-S' for template self-check (requires net access)" \ "- use '-W' to run browser-based, on-the-fly diff'ing test drive" \ "- use '-X' to show explanatory details about the upgrade" \ "- test specification can be granular, e.g. 'test2to3/022'" printf \ '\n%s\n %s\n %s\n %s\n %s\n %s\n %s\n %s\n' \ 'environment variables affecting the run + default/current values:' \ "- DIFF (${DIFF}): tool to compute and show differences of 2 files" \ "- DIFFOPTS (${DIFFOPTS}): options to the above tool" \ "- DIFFPAGER (${DIFFPAGER}): possibly accompanying the above tool" \ "- RNGVALIDATOR (${RNGVALIDATOR}): RelaxNG validator" \ "- XSLTPROCESSOR (${_XSLTPROCESSOR}): XSLT 1.0 capable processor" \ "- HTTPPORT (${HTTPPORT}): port used by test drive HTTP server run" \ "- WEBBROWSER (${WEBBROWSER}): used for in-browser test drive" } main() { _main_pass= _main_bailout=0 _main_ret=0 while test $# -gt 0; do case "$1" in -h) usage; exit;; -C|-G|-S|-X) _main_bailout=1;; esac _main_pass="${_main_pass} $1" shift done test_suite ${_main_pass} || _main_ret=$? test ${_main_bailout} -ne 0 \ || test_suite -C ${_main_pass} >/dev/null || true test ${_main_ret} = 0 && emit_result ${_main_ret} "Overall suite" \ || emit_result "at least 2^$((_main_ret - 1))" "Overall suite" return ${_main_ret} } main "$@" diff --git a/xml/xslt_cibtr-2.rng b/xml/xslt_cibtr-2.rng index 6dbe775151..e7be9b6f53 100644 --- a/xml/xslt_cibtr-2.rng +++ b/xml/xslt_cibtr-2.rng @@ -1,15 +1,16 @@ - +