diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..2418181b --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +configure.ac export-subst diff --git a/Makefile.am b/Makefile.am index d5657687..0dee6075 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,203 +1,204 @@ # # Copyright (C) 2010-2018 Red Hat, Inc. All rights reserved. # # Author: Fabio M. Di Nitto # # This software licensed under GPL-2.0+, LGPL-2.0+ # MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure depcomp \ config.guess config.sub missing install-sh \ ltmain.sh compile config.h.in config.h.in~ \ autoscan.log configure.scan test-driver \ m4/libtool.m4 m4/lt~obsolete.m4 m4/ltoptions.m4 \ m4/ltsugar.m4 m4/ltversion.m4 include $(top_srcdir)/build-aux/check.mk AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I m4 SPEC = $(PACKAGE_NAME).spec TARGZFILE = $(PACKAGE_NAME)-$(VERSION).tar.gz EXTRA_DIST = autogen.sh .version \ NOTES_TO_PACKAGE_MAINTAINERS \ $(SPEC).in build-aux SUBDIRS = init libtap libknet kronosnetd if BUILD_MAN SUBDIRS += man endif if BUILD_POC SUBDIRS += poc-code endif dist_doc_DATA = \ COPYING.applications \ COPYING.libraries \ COPYRIGHT \ README.licence \ README all-local: $(SPEC) clean-local: rm -f $(SPEC) distclean-local: rm -f $(PACKAGE_NAME)-*.tar.* $(PACKAGE_NAME)-*.sha256* tag-* ## make rpm/srpm section. $(SPEC): $(SPEC).in .version config.status rm -f $@-t $@ - date="`LC_ALL=C date "+%a %b %d %Y"`" && \ + date="`LC_ALL=C $(UTC_DATE_AT)$(SOURCE_EPOCH) "+%a %b %d %Y"`" && \ if [ -f $(abs_srcdir)/.tarball-version ]; then \ gitver="`cat $(abs_srcdir)/.tarball-version`" && \ rpmver=$$gitver && \ alphatag="" && \ dirty="" && \ numcomm="0"; \ else \ gitver="`GIT_DIR=$(abs_srcdir)/.git git describe --abbrev=4 --match='v*' HEAD 2>/dev/null`" && \ rpmver=`echo $$gitver | sed -e "s/^v//" -e "s/-.*//g"` && \ alphatag=`echo $$gitver | sed -e "s/.*-//" -e "s/^g//"` && \ vtag=`echo $$gitver | sed -e "s/-.*//g"` && \ numcomm=`GIT_DIR=$(abs_srcdir)/.git git rev-list $$vtag..HEAD | wc -l` && \ cd $(abs_srcdir) && \ git update-index --refresh > /dev/null 2>&1 || true && \ dirty=`git diff-index --name-only HEAD 2>/dev/null` && cd - 2>/dev/null; \ fi && \ if [ -n "$$dirty" ]; then dirty="dirty"; else dirty=""; fi && \ if [ "$$numcomm" = "0" ]; then \ sed \ -e "s#@version@#$$rpmver#g" \ -e "s#%glo.*alpha.*##g" \ -e "s#%glo.*numcomm.*##g" \ -e "s#@dirty@#$$dirty#g" \ -e "s#@date@#$$date#g" \ $(abs_srcdir)/$@.in > $@-t; \ else \ sed \ -e "s#@version@#$$rpmver#g" \ -e "s#@alphatag@#$$alphatag#g" \ -e "s#@numcomm@#$$numcomm#g" \ -e "s#@dirty@#$$dirty#g" \ -e "s#@date@#$$date#g" \ $(abs_srcdir)/$@.in > $@-t; \ fi; \ if [ -z "$$dirty" ]; then sed -i -e "s#%glo.*dirty.*##g" $@-t; fi if BUILD_SCTP sed -i -e "s#@sctp@#bcond_without#g" $@-t else sed -i -e "s#@sctp@#bcond_with#g" $@-t endif if BUILD_CRYPTO_NSS sed -i -e "s#@nss@#bcond_without#g" $@-t else sed -i -e "s#@nss@#bcond_with#g" $@-t endif if BUILD_CRYPTO_OPENSSL sed -i -e "s#@openssl@#bcond_without#g" $@-t else sed -i -e "s#@openssl@#bcond_with#g" $@-t endif if BUILD_COMPRESS_ZLIB sed -i -e "s#@zlib@#bcond_without#g" $@-t else sed -i -e "s#@zlib@#bcond_with#g" $@-t endif if BUILD_COMPRESS_LZ4 sed -i -e "s#@lz4@#bcond_without#g" $@-t else sed -i -e "s#@lz4@#bcond_with#g" $@-t endif if BUILD_COMPRESS_LZO2 sed -i -e "s#@lzo2@#bcond_without#g" $@-t else sed -i -e "s#@lzo2@#bcond_with#g" $@-t endif if BUILD_COMPRESS_LZMA sed -i -e "s#@lzma@#bcond_without#g" $@-t else sed -i -e "s#@lzma@#bcond_with#g" $@-t endif if BUILD_COMPRESS_BZIP2 sed -i -e "s#@bzip2@#bcond_without#g" $@-t else sed -i -e "s#@bzip2@#bcond_with#g" $@-t endif if BUILD_KRONOSNETD sed -i -e "s#@kronosnetd@#bcond_without#g" $@-t else sed -i -e "s#@kronosnetd@#bcond_with#g" $@-t endif if BUILD_LIBTAP sed -i -e "s#@libtap@#bcond_without#g" $@-t else sed -i -e "s#@libtap@#bcond_with#g" $@-t endif if BUILD_RUNAUTOGEN sed -i -e "s#@runautogen@#bcond_without#g" $@-t else sed -i -e "s#@runautogen@#bcond_with#g" $@-t endif if OVERRIDE_RPM_DEBUGINFO sed -i -e "s#@overriderpmdebuginfo@#bcond_without#g" $@-t else sed -i -e "s#@overriderpmdebuginfo@#bcond_with#g" $@-t endif if BUILD_RPM_DEBUGINFO sed -i -e "s#@rpmdebuginfo@#bcond_without#g" $@-t else sed -i -e "s#@rpmdebuginfo@#bcond_with#g" $@-t endif if BUILD_MAN sed -i -e "s#@buildman@#bcond_without#g" $@-t else sed -i -e "s#@buildman@#bcond_with#g" $@-t endif sed -i -e "s#@defaultadmgroup@#$(DEFAULTADMGROUP)#g" $@-t chmod a-w $@-t mv $@-t $@ rm -f $@-t* $(TARGZFILE): $(MAKE) dist RPMBUILDOPTS = --define "_sourcedir $(abs_builddir)" \ --define "_specdir $(abs_builddir)" \ --define "_builddir $(abs_builddir)" \ --define "_srcrpmdir $(abs_builddir)" \ --define "_rpmdir $(abs_builddir)" srpm: clean $(MAKE) $(SPEC) $(TARGZFILE) rpmbuild $(RPMBUILDOPTS) --nodeps -bs $(SPEC) rpm: clean $(MAKE) $(SPEC) $(TARGZFILE) rpmbuild $(RPMBUILDOPTS) -ba $(SPEC) # release/versioning BUILT_SOURCES = .version .version: echo $(VERSION) > $@-t && mv $@-t $@ dist-hook: gen-ChangeLog echo $(VERSION) > $(distdir)/.tarball-version + echo $(SOURCE_EPOCH) > $(distdir)/source_epoch gen_start_date = 2000-01-01 .PHONY: gen-ChangeLog gen-ChangeLog: if test -d $(abs_srcdir)/.git; then \ LC_ALL=C $(top_srcdir)/build-aux/gitlog-to-changelog \ --since=$(gen_start_date) > $(distdir)/cl-t; \ rm -f $(distdir)/ChangeLog; \ mv $(distdir)/cl-t $(distdir)/ChangeLog; \ fi diff --git a/build-aux/knet_valgrind_memcheck.supp b/build-aux/knet_valgrind_memcheck.supp index de0d2188..149aba35 100644 --- a/build-aux/knet_valgrind_memcheck.supp +++ b/build-aux/knet_valgrind_memcheck.supp @@ -1,543 +1,575 @@ { lzma internal stuff Memcheck:Cond fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_stream_header_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_crc32 fun:lzma_stream_header_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_crc32 fun:lzma_stream_header_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_block_header_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:is_overlap fun:memcpy@@GLIBC_2.14 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread } { lzma internal stuff Memcheck:Cond fun:memcpy@@GLIBC_2.14 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread } { lzma internal stuff Memcheck:Value8 fun:memcpy@@GLIBC_2.14 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread } { lzma internal stuff Memcheck:Value8 fun:lzma_vli_encode fun:lzma_filter_flags_encode fun:lzma_block_header_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_filter_flags_encode fun:lzma_block_header_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:memset fun:lzma_block_header_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:memset fun:lzma_block_header_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_block_unpadded_size fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_index_append fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_vli_size fun:lzma_index_append fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_vli_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_vli_encode obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_crc32 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_crc32 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_index_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_stream_footer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Value8 fun:lzma_crc32 fun:lzma_stream_footer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond fun:lzma_crc32 fun:lzma_stream_footer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread fun:clone } { lzma internal stuff Memcheck:Cond obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 obj:/usr/lib64/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread } { lzma internal stuff (Debian Unstable) Memcheck:Cond obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 fun:lzma_block_buffer_encode fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread } +{ + lzma internal stuff (Debian / Ubuntu) + Memcheck:Cond + obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.2.2 + fun:lzma_block_buffer_encode + fun:lzma_stream_buffer_encode + fun:lzma_easy_buffer_encode + fun:lzma_compress + fun:compress_lib_test + fun:compress_cfg + fun:knet_handle_compress + fun:test +} { lzma internal stuff (Ubuntu 17.10 i386) Memcheck:Cond obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread fun:start_thread } { lzma internal stuff (Ubuntu 17.10 i386) Memcheck:Cond obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 fun:lzma_stream_buffer_encode fun:lzma_easy_buffer_encode fun:lzma_compress fun:compress fun:_parse_recv_from_sock fun:_handle_send_to_links fun:_handle_send_to_links_thread } +{ + lzma internal stuff (Ubuntu 17.10 i386) + Memcheck:Cond + obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 + obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 + obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 + obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 + obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 + obj:/lib/i386-linux-gnu/liblzma.so.5.2.2 + fun:lzma_stream_buffer_encode + fun:lzma_easy_buffer_encode + fun:lzma_compress + fun:compress_lib_test + fun:compress_cfg + fun:knet_handle_compress +} { nss internal stuff (FreeBSD 11.1) Memcheck:Addr8 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 fun:PR_LoadLibraryWithFlags obj:/usr/local/lib/nss/libnssutil3.so fun:PORT_LoadLibraryFromOrigin obj:/usr/local/lib/nss/libnss3.so fun:PR_CallOnce obj:/usr/local/lib/nss/libnss3.so } { nss internal stuff (FreeBSD 11.1) Memcheck:Addr8 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 fun:PR_LoadLibraryWithFlags obj:/usr/local/lib/nss/libnssutil3.so fun:PORT_LoadLibraryFromOrigin obj:/usr/local/lib/nss/libnss3.so fun:PR_CallOnce obj:/usr/local/lib/nss/libnss3.so fun:SECMOD_LoadModule obj:/usr/local/lib/nss/libnss3.so } { nss internal stuff (FreeBSD 11.1) Memcheck:Addr8 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/libexec/ld-elf.so.1 obj:/usr/local/lib/libcrypto.so.10 fun:DSO_load fun:DSO_dsobyaddr obj:/usr/local/lib/libcrypto.so.10 fun:pthread_once fun:CRYPTO_THREAD_run_once fun:OPENSSL_init_crypto fun:opensslcrypto_init } diff --git a/configure.ac b/configure.ac index bd122016..85ac60c9 100644 --- a/configure.ac +++ b/configure.ac @@ -1,474 +1,528 @@ # # Copyright (C) 2010-2018 Red Hat, Inc. All rights reserved. # # Authors: Fabio M. Di Nitto # Federico Simoncelli # # This software licensed under GPL-2.0+, LGPL-2.0+ # # -*- Autoconf -*- # Process this file with autoconf to produce a configure script. # AC_PREREQ([2.63]) AC_INIT([kronosnet], m4_esyscmd([build-aux/git-version-gen .tarball-version]), [devel@lists.kronosnet.org]) # Don't let AC_PROC_CC (invoked by AC_USE_SYSTEM_EXTENSIONS) replace # undefined CFLAGS with -g -O2, overriding our special OPT_CFLAGS. : ${CFLAGS=""} AC_USE_SYSTEM_EXTENSIONS AM_INIT_AUTOMAKE([1.13 dist-bzip2 dist-xz color-tests -Wno-portability subdir-objects]) LT_PREREQ([2.2.6]) # --enable-new-dtags: Use RUNPATH instead of RPATH. # It is necessary to have this done before libtool does linker detection. # See also: https://github.com/kronosnet/kronosnet/issues/107 # --as-needed: Modern systems have builtin ceil() making -lm superfluous but # AC_SEARCH_LIBS can't detect this because it tests with a false prototype AX_CHECK_LINK_FLAG([-Wl,--enable-new-dtags], [AM_LDFLAGS=-Wl,--enable-new-dtags], [AC_MSG_ERROR(["Linker support for --enable-new-dtags is required"])]) AX_CHECK_LINK_FLAG([-Wl,--as-needed], [AM_LDFLAGS="$AM_LDFLAGS -Wl,--as-needed"]) AC_SUBST([AM_LDFLAGS]) saved_LDFLAGS="$LDFLAGS" LDFLAGS="$AM_LDFLAGS $LDFLAGS" LT_INIT LDFLAGS="$saved_LDFLAGS" AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_SRCDIR([kronosnetd/main.c]) AC_CONFIG_HEADERS([config.h]) AC_CANONICAL_HOST AC_LANG([C]) systemddir=${prefix}/lib/systemd/system if test "$prefix" = "NONE"; then prefix="/usr" if test "$localstatedir" = "\${prefix}/var"; then localstatedir="/var" fi if test "$sysconfdir" = "\${prefix}/etc"; then sysconfdir="/etc" fi if test "$systemddir" = "NONE/lib/systemd/system"; then systemddir=/lib/systemd/system fi if test "$libdir" = "\${exec_prefix}/lib"; then if test -e /usr/lib64; then libdir="/usr/lib64" else libdir="/usr/lib" fi fi fi AC_PROG_AWK AC_PROG_GREP AC_PROG_SED AC_PROG_CPP AC_PROG_CC AC_PROG_CC_C99 if test "x$ac_cv_prog_cc_c99" = "xno"; then AC_MSG_ERROR(["C99 support is required"]) fi AC_PROG_LN_S AC_PROG_INSTALL AC_PROG_MAKE_SET PKG_PROG_PKG_CONFIG AC_CHECK_PROGS([VALGRIND_EXEC], [valgrind]) AM_CONDITIONAL([HAS_VALGRIND], [test x$VALGRIND_EXEC != "x"]) # KNET_OPTION_DEFINES(stem,type,detection code) # stem: enters name of option, Automake conditional and preprocessor define # type: compress or crypto, determines where the default comes from AC_DEFUN([KNET_OPTION_DEFINES],[ AC_ARG_ENABLE([$2-$1],[AS_HELP_STRING([--disable-$2-$1],[disable libknet $1 support])],, [enable_$2_$1="$enable_$2_all"]) AM_CONDITIONAL([BUILD_]m4_toupper([$2_$1]),[test "x$enable_$2_$1" = xyes]) if test "x$enable_$2_$1" = xyes; then $3 fi AC_DEFINE_UNQUOTED([WITH_]m4_toupper([$2_$1]), [`test "x$enable_$2_$1" != xyes; echo $?`], $1 $2 [built in]) ]) AC_ARG_ENABLE([man], [AS_HELP_STRING([--disable-man],[disable man page creation])],, [ enable_man="yes" ]) AM_CONDITIONAL([BUILD_MAN], [test x$enable_man = xyes]) AC_ARG_ENABLE([libknet-sctp], [AS_HELP_STRING([--disable-libknet-sctp],[disable libknet SCTP support])],, [ enable_libknet_sctp="yes" ]) AM_CONDITIONAL([BUILD_SCTP], [test x$enable_libknet_sctp = xyes]) AC_ARG_ENABLE([crypto-all], [AS_HELP_STRING([--disable-crypto-all],[disable libknet all crypto modules support])],, [ enable_crypto_all="yes" ]) KNET_OPTION_DEFINES([nss],[crypto],[PKG_CHECK_MODULES([nss], [nss])]) KNET_OPTION_DEFINES([openssl],[crypto],[ PKG_CHECK_MODULES([openssl],[libcrypto < 1.1], [AC_DEFINE_UNQUOTED([BUILDCRYPTOOPENSSL10], [1], [openssl 1.0 crypto])], [PKG_CHECK_MODULES([openssl],[libcrypto >= 1.1], [AC_DEFINE_UNQUOTED([BUILDCRYPTOOPENSSL11], [1], [openssl 1.1 crypto])])]) ]) AC_ARG_ENABLE([compress-all], [AS_HELP_STRING([--disable-compress-all],[disable libknet all compress modules support])],, [ enable_compress_all="yes" ]) KNET_OPTION_DEFINES([zlib],[compress],[PKG_CHECK_MODULES([zlib], [zlib])]) KNET_OPTION_DEFINES([lz4],[compress],[PKG_CHECK_MODULES([liblz4], [liblz4])]) KNET_OPTION_DEFINES([lzo2],[compress],[ PKG_CHECK_MODULES([lzo2], [lzo2], [# work around broken pkg-config file in v2.10 AC_SUBST([lzo2_CFLAGS],[`echo $lzo2_CFLAGS | sed 's,/lzo *, ,'`])], [AC_CHECK_HEADERS([lzo/lzo1x.h], [AC_CHECK_LIB([lzo2], [lzo1x_decompress_safe], [AC_SUBST([lzo2_LIBS], [-llzo2])])], [AC_MSG_ERROR(["missing required lzo/lzo1x.h header"])])]) ]) KNET_OPTION_DEFINES([lzma],[compress],[PKG_CHECK_MODULES([liblzma], [liblzma])]) KNET_OPTION_DEFINES([bzip2],[compress],[ PKG_CHECK_MODULES([bzip2], [bzip2],, [AC_CHECK_HEADERS([bzlib.h], [AC_CHECK_LIB([bz2], [BZ2_bzBuffToBuffCompress], [AC_SUBST([bzip2_LIBS], [-lbz2])])], [AC_MSG_ERROR(["missing required bzlib.h"])])]) ]) AC_ARG_ENABLE([poc], [AS_HELP_STRING([--enable-poc],[enable building poc code])],, [ enable_poc="no" ]) AM_CONDITIONAL([BUILD_POC], [test x$enable_poc = xyes]) AC_ARG_ENABLE([kronosnetd], [AS_HELP_STRING([--enable-kronosnetd],[Kronosnetd support])],, [ enable_kronosnetd="no" ]) AM_CONDITIONAL([BUILD_KRONOSNETD], [test x$enable_kronosnetd = xyes]) AC_ARG_ENABLE([runautogen], [AS_HELP_STRING([--enable-runautogen],[run autogen.sh])],, [ enable_runautogen="no" ]) AM_CONDITIONAL([BUILD_RUNAUTOGEN], [test x$enable_runautogen = xyes]) override_rpm_debuginfo_option="yes" AC_ARG_ENABLE([rpm-debuginfo], [AS_HELP_STRING([--enable-rpm-debuginfo],[build debuginfo packages])],, [ enable_rpm_debuginfo="no", override_rpm_debuginfo_option="no" ]) AM_CONDITIONAL([BUILD_RPM_DEBUGINFO], [test x$enable_rpm_debuginfo = xyes]) AM_CONDITIONAL([OVERRIDE_RPM_DEBUGINFO], [test x$override_rpm_debuginfo_option = xyes]) AC_ARG_ENABLE([libtap], [AS_HELP_STRING([--enable-libtap],[libtap support])],, [ enable_libtap="no" ]) if test "x$enable_kronosnetd" = xyes; then enable_libtap=yes fi AM_CONDITIONAL([BUILD_LIBTAP], [test x$enable_libtap = xyes]) ## local helper functions # this function checks if CC support options passed as # args. Global CPPFLAGS are ignored during this test. cc_supports_flag() { saveCPPFLAGS="$CPPFLAGS" CPPFLAGS="$@" if echo $CC | grep -q clang; then CPPFLAGS="-Werror $CPPFLAGS" fi AC_MSG_CHECKING([whether $CC supports "$@"]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([])], [RC=0; AC_MSG_RESULT([yes])], [RC=1; AC_MSG_RESULT([no])]) CPPFLAGS="$saveCPPFLAGS" return $RC } # Checks for libraries. AX_PTHREAD(,[AC_MSG_ERROR([POSIX threads support is required])]) saved_LIBS="$LIBS" LIBS= AC_SEARCH_LIBS([ceil], [m], , [AC_MSG_ERROR([ceil not found])]) AC_SUBST([m_LIBS], [$LIBS]) LIBS= AC_SEARCH_LIBS([clock_gettime], [rt], , [AC_MSG_ERROR([clock_gettime not found])]) AC_SUBST([rt_LIBS], [$LIBS]) LIBS= AC_SEARCH_LIBS([dlopen], [dl dld], , [AC_MSG_ERROR([dlopen not found])]) AC_SUBST([dl_LIBS], [$LIBS]) LIBS="$saved_LIBS" # OS detection AC_MSG_CHECKING([for os in ${host_os}]) case "$host_os" in *linux*) AC_DEFINE_UNQUOTED([KNET_LINUX], [1], [Compiling for Linux platform]) AC_MSG_RESULT([Linux]) ;; *bsd*) AC_DEFINE_UNQUOTED([KNET_BSD], [1], [Compiling for BSD platform]) AC_MSG_RESULT([BSD]) if test "x$enable_libtap" = xyes; then AC_MSG_ERROR([libtap is not currently supported on BSD platforms]) fi ;; *) AC_MSG_ERROR([Unsupported OS? hmmmm]) ;; esac # Checks for header files. AC_CHECK_HEADERS([sys/epoll.h]) AC_CHECK_FUNCS([kevent]) # if neither sys/epoll.h nor kevent are present, we should fail. if test "x$ac_cv_header_sys_epoll_h" = xno && test "x$ac_cv_func_kevent" = xno; then AC_MSG_ERROR([Both epoll and kevent unavailable on this OS]) fi if test "x$ac_cv_header_sys_epoll_h" = xyes && test "x$ac_cv_func_kevent" = xyes; then AC_MSG_ERROR([Both epoll and kevent available on this OS, please contact the maintainers to fix the code]) fi if test "x$enable_libknet_sctp" = xyes; then AC_CHECK_HEADERS([netinet/sctp.h],, [AC_MSG_ERROR(["missing required SCTP headers"])]) fi # Checks for typedefs, structures, and compiler characteristics. AC_C_INLINE AC_TYPE_PID_T AC_TYPE_SIZE_T AC_TYPE_SSIZE_T AC_TYPE_UINT8_T AC_TYPE_UINT16_T AC_TYPE_UINT32_T AC_TYPE_UINT64_T AC_TYPE_INT8_T AC_TYPE_INT16_T AC_TYPE_INT32_T AC_TYPE_INT64_T if test "x$enable_man" = "xyes"; then AC_ARG_VAR([DOXYGEN], [override doxygen executable]) AC_CHECK_PROGS([DOXYGEN], [doxygen], [no]) if test "x$DOXYGEN" = xno; then AC_MSG_ERROR(["Doxygen command not found"]) fi # required by doxyxml to build man pages dynamically # Don't let AC_PROC_CC (invoked by AX_PROG_CC_FOR_BUILD) replace # undefined CFLAGS_FOR_BUILD with -g -O2, overriding our special OPT_CFLAGS. : ${CFLAGS_FOR_BUILD=""} AX_PROG_CC_FOR_BUILD saved_PKG_CONFIG="$PKG_CONFIG" saved_ac_cv_path_PKG_CONFIG="$ac_cv_path_PKG_CONFIG" unset PKG_CONFIG ac_cv_path_PKG_CONFIG AC_PATH_PROG([PKG_CONFIG], [pkg-config]) PKG_CHECK_MODULES([libqb_BUILD], [libqb]) PKG_CHECK_MODULES([libxml_BUILD], [libxml-2.0]) PKG_CONFIG="$saved_PKG_CONFIG" ac_cv_path_PKG_CONFIG="$saved_ac_cv_path_PKG_CONFIG" fi # checks (for kronosnetd) if test "x$enable_kronosnetd" = xyes; then AC_CHECK_HEADERS([security/pam_appl.h], [AC_CHECK_LIB([pam], [pam_start], [AC_SUBST([pam_LIBS], [-lpam])], [AC_MSG_ERROR([Unable to find LinuxPAM devel files])])]) AC_CHECK_HEADERS([security/pam_misc.h], [AC_CHECK_LIB([pam_misc], [misc_conv], [AC_SUBST([pam_misc_LIBS], [-lpam_misc])], [AC_MSG_ERROR([Unable to find LinuxPAM MISC devel files])])]) PKG_CHECK_MODULES([libqb], [libqb]) AC_CHECK_LIB([qb], [qb_log_thread_priority_set], [have_qb_log_thread_priority_set="yes"], [have_qb_log_thread_priority_set="no"]) if test "x${have_qb_log_thread_priority_set}" = xyes; then AC_DEFINE_UNQUOTED([HAVE_QB_LOG_THREAD_PRIORITY_SET], [1], [have qb_log_thread_priority_set]) fi fi # local options AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug],[enable debug build])]) AC_ARG_WITH([initdefaultdir], [AS_HELP_STRING([--with-initdefaultdir=DIR],[path to /etc/sysconfig or /etc/default dir])], [ INITDEFAULTDIR="$withval" ], [ INITDEFAULTDIR="$sysconfdir/default" ]) AC_ARG_WITH([initddir], [AS_HELP_STRING([--with-initddir=DIR],[path to init script directory])], [ INITDDIR="$withval" ], [ INITDDIR="$sysconfdir/init.d" ]) AC_ARG_WITH([systemddir], [AS_HELP_STRING([--with-systemddir=DIR],[path to systemd unit files directory])], [ SYSTEMDDIR="$withval" ], [ SYSTEMDDIR="$systemddir" ]) AC_ARG_WITH([syslogfacility], [AS_HELP_STRING([--with-syslogfacility=FACILITY],[default syslog facility])], [ SYSLOGFACILITY="$withval" ], [ SYSLOGFACILITY="LOG_DAEMON" ]) AC_ARG_WITH([sysloglevel], [AS_HELP_STRING([--with-sysloglevel=LEVEL],[default syslog level])], [ SYSLOGLEVEL="$withval" ], [ SYSLOGLEVEL="LOG_INFO" ]) AC_ARG_WITH([defaultadmgroup], [AS_HELP_STRING([--with-defaultadmgroup=GROUP], [define PAM group. Users part of this group will be allowed to configure kronosnet. Others will only receive read-only rights.])], [ DEFAULTADMGROUP="$withval" ], [ DEFAULTADMGROUP="kronosnetadm" ]) ## random vars LOGDIR=${localstatedir}/log/ RUNDIR=${localstatedir}/run/ DEFAULT_CONFIG_DIR=${sysconfdir}/kronosnet ## do subst AC_SUBST([DEFAULT_CONFIG_DIR]) AC_SUBST([INITDEFAULTDIR]) AC_SUBST([INITDDIR]) AC_SUBST([SYSTEMDDIR]) AC_SUBST([LOGDIR]) AC_SUBST([DEFAULTADMGROUP]) AC_DEFINE_UNQUOTED([DEFAULT_CONFIG_DIR], ["$(eval echo ${DEFAULT_CONFIG_DIR})"], [Default config directory]) AC_DEFINE_UNQUOTED([DEFAULT_CONFIG_FILE], ["$(eval echo ${DEFAULT_CONFIG_DIR}/kronosnetd.conf)"], [Default config file]) AC_DEFINE_UNQUOTED([LOGDIR], ["$(eval echo ${LOGDIR})"], [Default logging directory]) AC_DEFINE_UNQUOTED([DEFAULT_LOG_FILE], ["$(eval echo ${LOGDIR}/kronosnetd.log)"], [Default log file]) AC_DEFINE_UNQUOTED([RUNDIR], ["$(eval echo ${RUNDIR})"], [Default run directory]) AC_DEFINE_UNQUOTED([SYSLOGFACILITY], [$(eval echo ${SYSLOGFACILITY})], [Default syslog facility]) AC_DEFINE_UNQUOTED([SYSLOGLEVEL], [$(eval echo ${SYSLOGLEVEL})], [Default syslog level]) AC_DEFINE_UNQUOTED([DEFAULTADMGROUP], ["$(eval echo ${DEFAULTADMGROUP})"], [Default admin group]) # debug build stuff if test "x${enable_debug}" = xyes; then AC_DEFINE_UNQUOTED([DEBUG], [1], [Compiling Debugging code]) OPT_CFLAGS="-O0" else OPT_CFLAGS="-O3" fi # gdb flags if test "x${GCC}" = xyes; then GDB_FLAGS="-ggdb3" else GDB_FLAGS="-g" fi # extra warnings EXTRA_WARNINGS="" WARNLIST=" all extra unused shadow missing-prototypes missing-declarations suggest-attribute=noreturn suggest-attribute=format strict-prototypes declaration-after-statement pointer-arith write-strings cast-align bad-function-cast missing-format-attribute float-equal format=2 format-signedness format-security format-nonliteral no-long-long unsigned-char gnu89-inline no-strict-aliasing error address cpp overflow parentheses sequence-point switch shift-overflow=2 overlength-strings retundent-decls init-self uninitialized unused-but-set-variable unused-function unused-result unused-value unused-variable unknown-pragmas no-unused-parameter " for j in $WARNLIST; do if cc_supports_flag -W$j; then EXTRA_WARNINGS="$EXTRA_WARNINGS -W$j"; fi done AC_SUBST([AM_CFLAGS],["$OPT_CFLAGS $GDB_FLAGS $EXTRA_WARNINGS"]) +AX_PROG_DATE +AS_IF([test "$ax_cv_prog_date_gnu_date:$ax_cv_prog_date_gnu_utc" = yes:yes], + [UTC_DATE_AT="date -u -d@"], + [AS_IF([test "x$ax_cv_prog_date_bsd_date" = xyes], + [UTC_DATE_AT="date -u -r"], + [AC_MSG_ERROR([date utility unable to convert epoch to UTC])])]) +AC_SUBST([UTC_DATE_AT]) + +AC_ARG_VAR([SOURCE_EPOCH],[last modification date of the source]) +AC_MSG_NOTICE([trying to determine source epoch]) +AC_MSG_CHECKING([for source epoch in \$SOURCE_EPOCH]) +AS_IF([test -n "$SOURCE_EPOCH"], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_CHECKING([for source epoch in source_epoch file]) + AS_IF([test -e "$srcdir/source_epoch"], + [read SOURCE_EPOCH <"$srcdir/source_epoch" + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_CHECKING([for source epoch baked in by gitattributes export-subst]) + SOURCE_EPOCH='$Format:%at$' # template for rewriting by git-archive + AS_CASE([$SOURCE_EPOCH], + [?Format:*], # was not rewritten + [AC_MSG_RESULT([no]) + AC_MSG_CHECKING([for source epoch in \$SOURCE_DATE_EPOCH]) + AS_IF([test "x$SOURCE_DATE_EPOCH" != x], + [SOURCE_EPOCH="$SOURCE_DATE_EPOCH" + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_CHECKING([whether git log can provide a source epoch]) + SOURCE_EPOCH=f${SOURCE_EPOCH#\$F} # convert into git log --pretty format + SOURCE_EPOCH=$(cd "$srcdir" && git log -1 --pretty=${SOURCE_EPOCH%$} 2>/dev/null) + AS_IF([test -n "$SOURCE_EPOCH"], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no, using current time and breaking reproducibility]) + SOURCE_EPOCH=$(date +%s)])])], + [AC_MSG_RESULT([yes])] + )]) + ]) +AC_MSG_NOTICE([using source epoch $($UTC_DATE_AT$SOURCE_EPOCH +'%F %T %Z')]) + AC_CONFIG_FILES([ Makefile init/Makefile libtap/Makefile libtap/libtap.pc kronosnetd/Makefile kronosnetd/kronosnetd.logrotate libknet/Makefile libknet/libknet.pc libknet/tests/Makefile man/Makefile man/Doxyfile-knet poc-code/Makefile poc-code/iov-hash/Makefile poc-code/access-list/Makefile ]) +if test "x$VERSION" = "xUNKNOWN"; then + AC_MSG_ERROR([m4_text_wrap([ + configure was unable to determine the source tree's current version. This + generally happens when using git archive (or the github download button) + generated tarball/zip file. In order to workaround this issue, either use git + clone https://github.com/kronosnet/kronosnet.git or use an official release + tarball, available at https://kronosnet.org/releases/. Alternatively you + can add a compatible version in a .tarball-version file at the top of the + source tree, wipe your autom4te.cache dir and generated configure, and rerun + autogen.sh. + ], [ ], [ ], [76])]) +fi + AC_OUTPUT diff --git a/libknet/compress.c b/libknet/compress.c index 60799c12..536b0439 100644 --- a/libknet/compress.c +++ b/libknet/compress.c @@ -1,430 +1,481 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include "internals.h" #include "compress.h" #include "compress_model.h" #include "logging.h" #include "threads_common.h" #include "common.h" /* * internal module switch data */ /* * DO NOT CHANGE MODEL_ID HERE OR ONWIRE COMPATIBILITY * WILL BREAK! * * Always add new items before the last NULL. */ compress_model_t compress_modules_cmds[] = { { "none" , 0, 0, 0, NULL }, { "zlib" , 1, WITH_COMPRESS_ZLIB , 0, NULL }, { "lz4" , 2, WITH_COMPRESS_LZ4 , 0, NULL }, { "lz4hc", 3, WITH_COMPRESS_LZ4 , 0, NULL }, { "lzo2" , 4, WITH_COMPRESS_LZO2 , 0, NULL }, { "lzma" , 5, WITH_COMPRESS_LZMA , 0, NULL }, { "bzip2", 6, WITH_COMPRESS_BZIP2, 0, NULL }, { NULL, 255, 0, 0, NULL } }; static int max_model = 0; static struct timespec last_load_failure; static int compress_get_model(const char *model) { int idx = 0; while (compress_modules_cmds[idx].model_name != NULL) { if (!strcmp(compress_modules_cmds[idx].model_name, model)) { return compress_modules_cmds[idx].model_id; } idx++; } return -1; } static int compress_get_max_model(void) { int idx = 0; while (compress_modules_cmds[idx].model_name != NULL) { idx++; } return idx - 1; } static int compress_is_valid_model(int compress_model) { int idx = 0; while (compress_modules_cmds[idx].model_name != NULL) { if ((compress_model == compress_modules_cmds[idx].model_id) && (compress_modules_cmds[idx].built_in == 1)) { return 0; } idx++; } return -1; } static int val_level( knet_handle_t knet_h, int compress_model, int compress_level) { - return compress_modules_cmds[compress_model].ops->val_level(knet_h, compress_level); + if (compress_modules_cmds[compress_model].ops->val_level != NULL) { + return compress_modules_cmds[compress_model].ops->val_level(knet_h, compress_level); + } + return 0; } /* * compress_check_lib_is_init needs to be invoked in a locked context! */ static int compress_check_lib_is_init(knet_handle_t knet_h, int cmp_model) { /* * lack of a .is_init function means that the module does not require * init per handle so we use a fake reference in the compress_int_data * to identify that we already increased the libref for this handle */ if (compress_modules_cmds[cmp_model].loaded == 1) { if (compress_modules_cmds[cmp_model].ops->is_init == NULL) { if (knet_h->compress_int_data[cmp_model] != NULL) { return 1; } } else { if (compress_modules_cmds[cmp_model].ops->is_init(knet_h, cmp_model) == 1) { return 1; } } } return 0; } /* * compress_load_lib should _always_ be invoked in write lock context */ static int compress_load_lib(knet_handle_t knet_h, int cmp_model, int rate_limit) { struct timespec clock_now; unsigned long long timediff; /* * checking again for paranoia and because * compress_check_lib_is_init is usually invoked in read context * and we need to switch from read to write locking in between. * another thread might have init the library in the meantime */ if (compress_check_lib_is_init(knet_h, cmp_model)) { return 0; } /* * due to the fact that decompress can load libraries * on demand, depending on the compress model selected * on other nodes, it is possible for an attacker * to send crafted packets to attempt to load libraries * at random in a DoS fashion. * If there is an error loading a library, then we want * to rate_limit a retry to reload the library every X * seconds to avoid a lock DoS that could greatly slow * down libknet. */ if (rate_limit) { if ((last_load_failure.tv_sec != 0) || (last_load_failure.tv_nsec != 0)) { clock_gettime(CLOCK_MONOTONIC, &clock_now); timespec_diff(last_load_failure, clock_now, &timediff); if (timediff < 10000000000) { errno = EAGAIN; return -1; } } } if (compress_modules_cmds[cmp_model].loaded == 0) { compress_modules_cmds[cmp_model].ops = load_module (knet_h, "compress", compress_modules_cmds[cmp_model].model_name); if (!compress_modules_cmds[cmp_model].ops) { clock_gettime(CLOCK_MONOTONIC, &last_load_failure); return -1; } if (compress_modules_cmds[cmp_model].ops->abi_ver != KNET_COMPRESS_MODEL_ABI) { log_err(knet_h, KNET_SUB_COMPRESS, "ABI mismatch loading module %s. knet ver: %d, module ver: %d", compress_modules_cmds[cmp_model].model_name, KNET_COMPRESS_MODEL_ABI, compress_modules_cmds[cmp_model].ops->abi_ver); errno = EINVAL; return -1; } compress_modules_cmds[cmp_model].loaded = 1; } if (compress_modules_cmds[cmp_model].ops->init != NULL) { if (compress_modules_cmds[cmp_model].ops->init(knet_h, cmp_model) < 0) { return -1; } } else { knet_h->compress_int_data[cmp_model] = (void *)&"1"; } return 0; } +static int compress_lib_test(knet_handle_t knet_h) +{ + int savederrno = 0; + unsigned char src[KNET_DATABUFSIZE]; + unsigned char dst[KNET_DATABUFSIZE_COMPRESS]; + ssize_t dst_comp_len = KNET_DATABUFSIZE_COMPRESS, dst_decomp_len = KNET_DATABUFSIZE; + unsigned int i; + + memset(src, 0, KNET_DATABUFSIZE); + memset(dst, 0, KNET_DATABUFSIZE_COMPRESS); + + /* + * NOTE: we cannot use compress and decompress API calls due to locking + * so we need to call directly into the modules + */ + + if (compress_modules_cmds[knet_h->compress_model].ops->compress(knet_h, src, KNET_DATABUFSIZE, dst, &dst_comp_len) < 0) { + savederrno = errno; + log_err(knet_h, KNET_SUB_COMPRESS, "Unable to compress test buffer. Please check your compression settings: %s", strerror(savederrno)); + errno = savederrno; + return -1; + } + + if (compress_modules_cmds[knet_h->compress_model].ops->decompress(knet_h, dst, dst_comp_len, src, &dst_decomp_len) < 0) { + savederrno = errno; + log_err(knet_h, KNET_SUB_COMPRESS, "Unable to decompress test buffer. Please check your compression settings: %s", strerror(savederrno)); + errno = savederrno; + return -1; + } + + for (i = 0; i < KNET_DATABUFSIZE; i++) { + if (src[i] != 0) { + log_err(knet_h, KNET_SUB_COMPRESS, "Decompressed buffer contains incorrect data"); + errno = EINVAL; + return -1; + } + } + + return 0; +} + int compress_init( knet_handle_t knet_h) { max_model = compress_get_max_model(); if (max_model > KNET_MAX_COMPRESS_METHODS) { log_err(knet_h, KNET_SUB_COMPRESS, "Too many compress methods defined in compress.c."); errno = EINVAL; return -1; } memset(&last_load_failure, 0, sizeof(struct timespec)); return 0; } int compress_cfg( knet_handle_t knet_h, struct knet_handle_compress_cfg *knet_handle_compress_cfg) { int savederrno = 0, err = 0; int cmp_model; cmp_model = compress_get_model(knet_handle_compress_cfg->compress_model); if (cmp_model < 0) { log_err(knet_h, KNET_SUB_COMPRESS, "compress model %s not supported", knet_handle_compress_cfg->compress_model); errno = EINVAL; return -1; } log_debug(knet_h, KNET_SUB_COMPRESS, "Initizializing compress module [%s/%d/%u]", knet_handle_compress_cfg->compress_model, knet_handle_compress_cfg->compress_level, knet_handle_compress_cfg->compress_threshold); if (cmp_model > 0) { if (compress_modules_cmds[cmp_model].built_in == 0) { log_err(knet_h, KNET_SUB_COMPRESS, "compress model %s support has not been built in. Please contact your vendor or fix the build", knet_handle_compress_cfg->compress_model); errno = EINVAL; return -1; } if (knet_handle_compress_cfg->compress_threshold > KNET_MAX_PACKET_SIZE) { log_err(knet_h, KNET_SUB_COMPRESS, "compress threshold cannot be higher than KNET_MAX_PACKET_SIZE (%d).", KNET_MAX_PACKET_SIZE); errno = EINVAL; return -1; } if (knet_handle_compress_cfg->compress_threshold == 0) { knet_h->compress_threshold = KNET_COMPRESS_THRESHOLD; log_debug(knet_h, KNET_SUB_COMPRESS, "resetting compression threshold to default (%d)", KNET_COMPRESS_THRESHOLD); } else { knet_h->compress_threshold = knet_handle_compress_cfg->compress_threshold; } savederrno = pthread_rwlock_rdlock(&shlib_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_COMPRESS, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (!compress_check_lib_is_init(knet_h, cmp_model)) { /* * need to switch to write lock, load the lib, and return with a write lock * this is not racy because compress_load_lib is written idempotent. */ pthread_rwlock_unlock(&shlib_rwlock); savederrno = pthread_rwlock_wrlock(&shlib_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_COMPRESS, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (compress_load_lib(knet_h, cmp_model, 0) < 0) { savederrno = errno; log_err(knet_h, KNET_SUB_COMPRESS, "Unable to load library: %s", strerror(savederrno)); err = -1; goto out_unlock; } } if (val_level(knet_h, cmp_model, knet_handle_compress_cfg->compress_level) < 0) { log_err(knet_h, KNET_SUB_COMPRESS, "compress level %d not supported for model %s", knet_handle_compress_cfg->compress_level, knet_handle_compress_cfg->compress_model); savederrno = EINVAL; err = -1; goto out_unlock; } + knet_h->compress_model = cmp_model; + knet_h->compress_level = knet_handle_compress_cfg->compress_level; + + if (compress_lib_test(knet_h) < 0) { + savederrno = errno; + err = -1; + goto out_unlock; + } + out_unlock: pthread_rwlock_unlock(&shlib_rwlock); } - if (!err) { - knet_h->compress_model = cmp_model; - knet_h->compress_level = knet_handle_compress_cfg->compress_level; - } else { + if (err) { knet_h->compress_model = 0; + knet_h->compress_level = 0; } errno = savederrno; return err; } void compress_fini( knet_handle_t knet_h, int all) { int savederrno = 0; int idx = 0; savederrno = pthread_rwlock_wrlock(&shlib_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_COMPRESS, "Unable to get write lock: %s", strerror(savederrno)); return; } while (compress_modules_cmds[idx].model_name != NULL) { if ((compress_modules_cmds[idx].built_in == 1) && (compress_modules_cmds[idx].loaded == 1) && (compress_modules_cmds[idx].model_id > 0) && (knet_h->compress_int_data[idx] != NULL) && (idx < KNET_MAX_COMPRESS_METHODS)) { if ((all) || (compress_modules_cmds[idx].model_id == knet_h->compress_model)) { if (compress_modules_cmds[idx].ops->fini != NULL) { compress_modules_cmds[idx].ops->fini(knet_h, idx); } else { knet_h->compress_int_data[idx] = NULL; } } } idx++; } pthread_rwlock_unlock(&shlib_rwlock); return; } /* * compress does not require compress_check_lib_is_init * because it's protected by compress_cfg */ int compress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { return compress_modules_cmds[knet_h->compress_model].ops->compress(knet_h, buf_in, buf_in_len, buf_out, buf_out_len); } int decompress( knet_handle_t knet_h, int compress_model, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int savederrno = 0, err = 0; if (compress_model > max_model) { log_err(knet_h, KNET_SUB_COMPRESS, "Received packet with unknown compress model %d", compress_model); errno = EINVAL; return -1; } if (compress_is_valid_model(compress_model) < 0) { log_err(knet_h, KNET_SUB_COMPRESS, "Received packet compressed with %s but support is not built in this version of libknet. Please contact your distribution vendor or fix the build.", compress_modules_cmds[compress_model].model_name); errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&shlib_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_COMPRESS, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (!compress_check_lib_is_init(knet_h, compress_model)) { /* * need to switch to write lock, load the lib, and return with a write lock * this is not racy because compress_load_lib is written idempotent. */ pthread_rwlock_unlock(&shlib_rwlock); savederrno = pthread_rwlock_wrlock(&shlib_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_COMPRESS, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (compress_load_lib(knet_h, compress_model, 1) < 0) { savederrno = errno; err = -1; log_err(knet_h, KNET_SUB_COMPRESS, "Unable to load library: %s", strerror(savederrno)); goto out_unlock; } } err = compress_modules_cmds[compress_model].ops->decompress(knet_h, buf_in, buf_in_len, buf_out, buf_out_len); savederrno = errno; out_unlock: pthread_rwlock_unlock(&shlib_rwlock); errno = savederrno; return err; } int knet_get_compress_list(struct knet_compress_info *compress_list, size_t *compress_list_entries) { int err = 0; int idx = 0; int outidx = 0; if (!compress_list_entries) { errno = EINVAL; return -1; } while (compress_modules_cmds[idx].model_name != NULL) { if (compress_modules_cmds[idx].built_in) { if (compress_list) { compress_list[outidx].name = compress_modules_cmds[idx].model_name; } outidx++; } idx++; } *compress_list_entries = outidx; return err; } diff --git a/libknet/compress_bzip2.c b/libknet/compress_bzip2.c index 597e03aa..5b47e1c9 100644 --- a/libknet/compress_bzip2.c +++ b/libknet/compress_bzip2.c @@ -1,126 +1,114 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #define KNET_MODULE #include "config.h" #include #include #include "logging.h" #include "compress_model.h" -static int bzip2_val_level( - knet_handle_t knet_h, - int compress_level) -{ - if ((compress_level < 1) || (compress_level > 9)) { - log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 unsupported compression level %d (accepted values from 1 to 9)", compress_level); - errno = EINVAL; - return -1; - } - return 0; -} - static int bzip2_compress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int err = 0; int savederrno = 0; unsigned int destLen = KNET_DATABUFSIZE_COMPRESS; err = BZ2_bzBuffToBuffCompress((char *)buf_out, &destLen, (char *)buf_in, buf_in_len, knet_h->compress_level, 0, 0); switch(err) { case BZ_OK: *buf_out_len = destLen; break; case BZ_MEM_ERROR: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 compress has not enough memory"); savederrno = ENOMEM; err = -1; break; case BZ_OUTBUFF_FULL: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 unable to compress source in destination buffer"); savederrno = E2BIG; err = -1; break; default: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 compress unknown error %d", err); savederrno = EINVAL; err = -1; break; } errno = savederrno; return err; } static int bzip2_decompress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int err = 0; int savederrno = 0; unsigned int destLen = KNET_DATABUFSIZE_COMPRESS; err = BZ2_bzBuffToBuffDecompress((char *)buf_out, &destLen, (char *)buf_in, buf_in_len, 0, 0); switch(err) { case BZ_OK: *buf_out_len = destLen; break; case BZ_MEM_ERROR: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 decompress has not enough memory"); savederrno = ENOMEM; err = -1; break; case BZ_OUTBUFF_FULL: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 unable to decompress source in destination buffer"); savederrno = E2BIG; err = -1; break; case BZ_DATA_ERROR: case BZ_DATA_ERROR_MAGIC: case BZ_UNEXPECTED_EOF: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 decompress detected input data corruption"); savederrno = EINVAL; err = -1; break; default: log_err(knet_h, KNET_SUB_BZIP2COMP, "bzip2 decompress unknown error %d", err); savederrno = EINVAL; err = -1; break; } errno = savederrno; return err; } compress_ops_t compress_model = { KNET_COMPRESS_MODEL_ABI, NULL, NULL, NULL, - bzip2_val_level, + NULL, bzip2_compress, bzip2_decompress }; diff --git a/libknet/compress_lz4.c b/libknet/compress_lz4.c index 8f90a5a9..263428a3 100644 --- a/libknet/compress_lz4.c +++ b/libknet/compress_lz4.c @@ -1,102 +1,91 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #define KNET_MODULE #include "config.h" #include #include #include "logging.h" #include "compress_model.h" -static int lz4_val_level( - knet_handle_t knet_h, - int compress_level) -{ - if (compress_level <= 0) { - log_info(knet_h, KNET_SUB_LZ4COMP, "lz4 acceleration level 0 (or negatives) are automatically remapped to 1"); - } - - return 0; -} - static int lz4_compress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int lzerr = 0, err = 0; int savederrno = 0; lzerr = LZ4_compress_fast((const char *)buf_in, (char *)buf_out, buf_in_len, KNET_DATABUFSIZE_COMPRESS, knet_h->compress_level); /* * data compressed */ if (lzerr > 0) { *buf_out_len = lzerr; } /* * unable to compress */ if (lzerr == 0) { *buf_out_len = buf_in_len; } /* * lz4 internal error */ if (lzerr < 0) { log_err(knet_h, KNET_SUB_LZ4COMP, "lz4 compression error: %d", lzerr); savederrno = EINVAL; err = -1; } errno = savederrno; return err; } static int lz4_decompress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int lzerr = 0, err = 0; int savederrno = 0; lzerr = LZ4_decompress_safe((const char *)buf_in, (char *)buf_out, buf_in_len, KNET_DATABUFSIZE); if (lzerr < 0) { log_err(knet_h, KNET_SUB_LZ4COMP, "lz4 decompression error: %d", lzerr); savederrno = EINVAL; err = -1; } if (lzerr > 0) { *buf_out_len = lzerr; } errno = savederrno; return err; } compress_ops_t compress_model = { KNET_COMPRESS_MODEL_ABI, NULL, NULL, NULL, - lz4_val_level, + NULL, lz4_compress, lz4_decompress }; diff --git a/libknet/compress_lz4hc.c b/libknet/compress_lz4hc.c index aa832af6..da40fbc9 100644 --- a/libknet/compress_lz4hc.c +++ b/libknet/compress_lz4hc.c @@ -1,123 +1,102 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #define KNET_MODULE #include "config.h" #include #include #include #include "logging.h" #include "compress_model.h" #ifdef LZ4HC_CLEVEL_MAX #define KNET_LZ4HC_MAX LZ4HC_CLEVEL_MAX #endif #ifdef LZ4HC_MAX_CLEVEL #define KNET_LZ4HC_MAX LZ4HC_MAX_CLEVEL #endif #ifndef KNET_LZ4HC_MAX /* * older releases of lz4 do not define LZ4HC_CLEVEL range. * According to lz4hc.h, any value between 0 and 16 is valid. * We defalt to 16 based on the comments in the include file * from older versions. */ #define KNET_LZ4HC_MAX 16 #endif -static int lz4hc_val_level( - knet_handle_t knet_h, - int compress_level) -{ - if (compress_level < 1) { - log_err(knet_h, KNET_SUB_LZ4HCCOMP, "lz4hc supports only 1+ values for compression level"); - errno = EINVAL; - return -1; - } - - if (compress_level < 4) { - log_info(knet_h, KNET_SUB_LZ4HCCOMP, "lz4hc recommends 4+ compression level for better results"); - } - - if (compress_level > KNET_LZ4HC_MAX) { - log_warn(knet_h, KNET_SUB_LZ4HCCOMP, "lz4hc installed on this system supports up to compression level %d. Higher values behaves as %d", KNET_LZ4HC_MAX, KNET_LZ4HC_MAX); - } - - return 0; -} - static int lz4hc_compress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int lzerr = 0, err = 0; int savederrno = 0; lzerr = LZ4_compress_HC((const char *)buf_in, (char *)buf_out, buf_in_len, KNET_DATABUFSIZE_COMPRESS, knet_h->compress_level); /* * data compressed */ if (lzerr > 0) { *buf_out_len = lzerr; } /* * unable to compress */ if (lzerr <= 0) { log_err(knet_h, KNET_SUB_LZ4HCCOMP, "lz4hc compression error: %d", lzerr); savederrno = EINVAL; err = -1; } errno = savederrno; return err; } /* This is a straight copy from compress_lz4.c */ static int lz4_decompress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int lzerr = 0, err = 0; int savederrno = 0; lzerr = LZ4_decompress_safe((const char *)buf_in, (char *)buf_out, buf_in_len, KNET_DATABUFSIZE); if (lzerr < 0) { log_err(knet_h, KNET_SUB_LZ4COMP, "lz4 decompression error: %d", lzerr); savederrno = EINVAL; err = -1; } if (lzerr > 0) { *buf_out_len = lzerr; } errno = savederrno; return err; } compress_ops_t compress_model = { KNET_COMPRESS_MODEL_ABI, NULL, NULL, NULL, - lz4hc_val_level, + NULL, lz4hc_compress, lz4_decompress }; diff --git a/libknet/compress_lzma.c b/libknet/compress_lzma.c index 2a6c356c..cd037403 100644 --- a/libknet/compress_lzma.c +++ b/libknet/compress_lzma.c @@ -1,138 +1,125 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #define KNET_MODULE #include "config.h" #include #include #include "logging.h" #include "compress_model.h" -static int lzma_val_level( - knet_handle_t knet_h, - int compress_level) -{ - if ((compress_level < 0) || (compress_level > 9)) { - log_err(knet_h, KNET_SUB_LZMACOMP, "lzma unsupported compression preset %d (accepted values from 0 to 9)", compress_level); - errno = EINVAL; - return -1; - } - - return 0; -} - static int lzma_compress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int err = 0; int savederrno = 0; size_t out_pos = 0; lzma_ret ret = 0; ret = lzma_easy_buffer_encode(knet_h->compress_level, LZMA_CHECK_NONE, NULL, (const uint8_t *)buf_in, buf_in_len, (uint8_t *)buf_out, &out_pos, KNET_DATABUFSIZE_COMPRESS); switch(ret) { case LZMA_OK: *buf_out_len = out_pos; break; case LZMA_MEM_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma compress memory allocation failed"); savederrno = ENOMEM; err = -1; break; case LZMA_MEMLIMIT_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma compress requires higher memory boundaries (see lzma_memlimit_set)"); savederrno = ENOMEM; err = -1; break; case LZMA_PROG_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma compress has been called with incorrect options"); savederrno = EINVAL; err = -1; break; default: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma compress unknown error %u", ret); savederrno = EINVAL; err = -1; break; } errno = savederrno; return err; } static int lzma_decompress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int err = 0; int savederrno = 0; uint64_t memlimit = UINT64_MAX; /* disable lzma internal memlimit check */ size_t out_pos = 0, in_pos = 0; lzma_ret ret = 0; ret = lzma_stream_buffer_decode(&memlimit, 0, NULL, (const uint8_t *)buf_in, &in_pos, buf_in_len, (uint8_t *)buf_out, &out_pos, KNET_DATABUFSIZE_COMPRESS); switch(ret) { case LZMA_OK: *buf_out_len = out_pos; break; case LZMA_MEM_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma decompress memory allocation failed"); savederrno = ENOMEM; err = -1; break; case LZMA_MEMLIMIT_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma decompress requires higher memory boundaries (see lzma_memlimit_set)"); savederrno = ENOMEM; err = -1; break; case LZMA_DATA_ERROR: case LZMA_FORMAT_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma decompress invalid data received"); savederrno = EINVAL; err = -1; break; case LZMA_PROG_ERROR: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma decompress has been called with incorrect options"); savederrno = EINVAL; err = -1; break; default: log_err(knet_h, KNET_SUB_LZMACOMP, "lzma decompress unknown error %u", ret); savederrno = EINVAL; err = -1; break; } errno = savederrno; return err; } compress_ops_t compress_model = { KNET_COMPRESS_MODEL_ABI, NULL, NULL, NULL, - lzma_val_level, + NULL, lzma_compress, lzma_decompress }; diff --git a/libknet/compress_model.h b/libknet/compress_model.h index 082cc5af..5d6a640e 100644 --- a/libknet/compress_model.h +++ b/libknet/compress_model.h @@ -1,91 +1,91 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #ifndef __KNET_COMPRESS_MODEL_H__ #define __KNET_COMPRESS_MODEL_H__ #include "internals.h" #define KNET_COMPRESS_MODEL_ABI 1 typedef struct { uint8_t abi_ver; /* * some libs need special init and handling of buffers etc. * is_init is called in shlib_rwlock read only context to see if * the module has been initialized within this knet_handle. * Providing is_init is optional. A module that does not export * an is_init and if the associated shared library is already loaded * is treated as "does not require init". */ int (*is_init) (knet_handle_t knet_h, int method_idx); /* * init is called when the library requires special init handling, * such as memory allocation and such. * init is invoked in shlib_rwlock write only context when * the module exports this function. * It is optional to provide an init function if the module * does not require any init. */ int (*init) (knet_handle_t knet_h, int method_idx); /* * fini is invoked only on knet_handle_free in a write only context. * It is optional to provide this function if the module * does not require any finalization */ void (*fini) (knet_handle_t knet_h, int method_idx); /* * runtime config validation and compress/decompress */ /* - * required functions - * * val_level is called upon compress configuration changes * to make sure that the requested compress_level is valid * within the context of a given module. */ int (*val_level)(knet_handle_t knet_h, int compress_level); /* + * required functions + * * hopefully those 2 don't require any explanation.... */ int (*compress) (knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len); int (*decompress)(knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len); } compress_ops_t; typedef struct { const char *model_name; uint8_t model_id; /* sequential unique identifier */ uint8_t built_in; /* set at configure/build time to 1 if available */ /* * library is loaded */ uint8_t loaded; /* * runtime bits */ compress_ops_t *ops; } compress_model_t; #endif diff --git a/libknet/compress_zlib.c b/libknet/compress_zlib.c index 56bbfc53..3f86ac14 100644 --- a/libknet/compress_zlib.c +++ b/libknet/compress_zlib.c @@ -1,136 +1,117 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #define KNET_MODULE #include "config.h" #include #include #include "logging.h" #include "compress_model.h" -static int zlib_val_level( - knet_handle_t knet_h, - int compress_level) -{ - if (compress_level < 0) { - log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib does not support negative compression level %d", - compress_level); - return -1; - } - if (compress_level > 9) { - log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib does not support compression level higher than 9"); - return -1; - } - if (compress_level == 0) { - log_warn(knet_h, KNET_SUB_ZLIBCOMP, "zlib compress level 0 does NOT perform any compression"); - } - return 0; -} - static int zlib_compress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int zerr = 0, err = 0; int savederrno = 0; uLongf destLen = *buf_out_len; zerr = compress2(buf_out, &destLen, buf_in, buf_in_len, knet_h->compress_level); *buf_out_len = destLen; switch(zerr) { case Z_OK: err = 0; savederrno = 0; break; case Z_MEM_ERROR: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib compress mem error"); err = -1; savederrno = ENOMEM; break; case Z_BUF_ERROR: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib compress buf error"); err = -1; savederrno = ENOBUFS; break; case Z_STREAM_ERROR: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib compress stream error"); err = -1; savederrno = EINVAL; break; default: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib unknown compress error: %d", zerr); break; } errno = savederrno; return err; } static int zlib_decompress( knet_handle_t knet_h, const unsigned char *buf_in, const ssize_t buf_in_len, unsigned char *buf_out, ssize_t *buf_out_len) { int zerr = 0, err = 0; int savederrno = 0; uLongf destLen = *buf_out_len; zerr = uncompress(buf_out, &destLen, buf_in, buf_in_len); *buf_out_len = destLen; switch(zerr) { case Z_OK: err = 0; savederrno = 0; break; case Z_MEM_ERROR: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib decompress mem error"); err = -1; savederrno = ENOMEM; break; case Z_BUF_ERROR: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib decompress buf error"); err = -1; savederrno = ENOBUFS; break; case Z_DATA_ERROR: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib decompress data error"); err = -1; savederrno = EINVAL; break; default: log_err(knet_h, KNET_SUB_ZLIBCOMP, "zlib unknown decompress error: %d", zerr); break; } errno = savederrno; return err; } compress_ops_t compress_model = { KNET_COMPRESS_MODEL_ABI, NULL, NULL, NULL, - zlib_val_level, + NULL, zlib_compress, zlib_decompress }; diff --git a/libknet/tests/api_knet_handle_compress.c b/libknet/tests/api_knet_handle_compress.c index d8f9714e..76a0ce67 100644 --- a/libknet/tests/api_knet_handle_compress.c +++ b/libknet/tests/api_knet_handle_compress.c @@ -1,181 +1,181 @@ /* * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include "libknet.h" #include "internals.h" #include "test-common.h" static void test(void) { knet_handle_t knet_h; int logfds[2]; struct knet_handle_compress_cfg knet_handle_compress_cfg; memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); printf("Test knet_handle_compress incorrect knet_h\n"); if ((!knet_handle_compress(NULL, &knet_handle_compress_cfg)) || (errno != EINVAL)) { printf("knet_handle_compress accepted invalid knet_h or returned incorrect error: %s\n", strerror(errno)); exit(FAIL); } setup_logpipes(logfds); knet_h = knet_handle_start(logfds, KNET_LOG_DEBUG); flush_logs(logfds[0], stdout); printf("Test knet_handle_compress with invalid cfg\n"); if ((!knet_handle_compress(knet_h, NULL)) || (errno != EINVAL)) { printf("knet_handle_compress accepted invalid cfg or returned incorrect error: %s\n", strerror(errno)); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); printf("Test knet_handle_compress with un-initialized cfg\n"); memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); if ((!knet_handle_compress(knet_h, &knet_handle_compress_cfg)) || (errno != EINVAL)) { printf("knet_handle_compress accepted invalid un-initialized cfg\n"); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); printf("Test knet_handle_compress with none compress model (disable compress)\n"); memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); strncpy(knet_handle_compress_cfg.compress_model, "none", sizeof(knet_handle_compress_cfg.compress_model) - 1); if (knet_handle_compress(knet_h, &knet_handle_compress_cfg) != 0) { printf("knet_handle_compress did not accept none compress mode cfg\n"); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); - printf("Test knet_handle_compress with zlib compress and negative level\n"); + printf("Test knet_handle_compress with zlib compress and negative level (-2)\n"); memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); strncpy(knet_handle_compress_cfg.compress_model, "zlib", sizeof(knet_handle_compress_cfg.compress_model) - 1); - knet_handle_compress_cfg.compress_level = -1; + knet_handle_compress_cfg.compress_level = -2; if ((!knet_handle_compress(knet_h, &knet_handle_compress_cfg)) || (errno != EINVAL)) { - printf("knet_handle_compress accepted invalid (-1) compress level for zlib\n"); + printf("knet_handle_compress accepted invalid (-2) compress level for zlib\n"); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); printf("Test knet_handle_compress with zlib compress and excessive compress level\n"); memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); strncpy(knet_handle_compress_cfg.compress_model, "zlib", sizeof(knet_handle_compress_cfg.compress_model) - 1); knet_handle_compress_cfg.compress_level = 10; if ((!knet_handle_compress(knet_h, &knet_handle_compress_cfg)) || (errno != EINVAL)) { printf("knet_handle_compress accepted invalid (10) compress level for zlib\n"); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); printf("Test knet_handle_compress with zlib compress and excessive compress threshold\n"); memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); strncpy(knet_handle_compress_cfg.compress_model, "zlib", sizeof(knet_handle_compress_cfg.compress_model) - 1); knet_handle_compress_cfg.compress_level = 1; knet_handle_compress_cfg.compress_threshold = KNET_MAX_PACKET_SIZE +1; if ((!knet_handle_compress(knet_h, &knet_handle_compress_cfg)) || (errno != EINVAL)) { printf("knet_handle_compress accepted invalid compress threshold\n"); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); printf("Test knet_handle_compress with zlib compress model normal compress level and threshold\n"); memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); strncpy(knet_handle_compress_cfg.compress_model, "zlib", sizeof(knet_handle_compress_cfg.compress_model) - 1); knet_handle_compress_cfg.compress_level = 1; knet_handle_compress_cfg.compress_threshold = 64; if (knet_handle_compress(knet_h, &knet_handle_compress_cfg) != 0) { printf("knet_handle_compress did not accept zlib compress mode with compress level 1 cfg\n"); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); exit(FAIL); } flush_logs(logfds[0], stdout); knet_handle_free(knet_h); flush_logs(logfds[0], stdout); close_logpipes(logfds); } int main(int argc, char *argv[]) { struct knet_compress_info compress_list[16]; size_t compress_list_entries; size_t i; memset(compress_list, 0, sizeof(compress_list)); if (knet_get_compress_list(compress_list, &compress_list_entries) < 0) { printf("knet_get_compress_list failed: %s\n", strerror(errno)); return FAIL; } if (compress_list_entries == 0) { printf("no compression modules detected. Skipping\n"); return SKIP; } for (i=0; i < compress_list_entries; i++) { if (!strcmp(compress_list[i].name, "zlib")) { test(); return PASS; } } printf("WARNING: zlib support not builtin the library. Unable to test/verify internal compress API calls\n"); return SKIP; } diff --git a/libknet/threads_rx.c b/libknet/threads_rx.c index 0e9ba432..3b047786 100644 --- a/libknet/threads_rx.c +++ b/libknet/threads_rx.c @@ -1,836 +1,840 @@ /* * Copyright (C) 2012-2018 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include "compat.h" #include "compress.h" #include "crypto.h" #include "host.h" #include "links.h" #include "logging.h" #include "transports.h" #include "transport_common.h" #include "threads_common.h" #include "threads_heartbeat.h" #include "threads_rx.h" #include "netutils.h" /* * RECV */ /* * return 1 if a > b * return -1 if b > a * return 0 if they are equal */ static inline int timecmp(struct timespec a, struct timespec b) { if (a.tv_sec != b.tv_sec) { if (a.tv_sec > b.tv_sec) { return 1; } else { return -1; } } else { if (a.tv_nsec > b.tv_nsec) { return 1; } else if (a.tv_nsec < b.tv_nsec) { return -1; } else { return 0; } } } /* * this functions needs to return an index (0 to 7) * to a knet_host_defrag_buf. (-1 on errors) */ static int find_pckt_defrag_buf(knet_handle_t knet_h, struct knet_header *inbuf) { struct knet_host *src_host = knet_h->host_index[inbuf->kh_node]; int i, oldest; /* * check if there is a buffer already in use handling the same seq_num */ for (i = 0; i < KNET_MAX_LINK; i++) { if (src_host->defrag_buf[i].in_use) { if (src_host->defrag_buf[i].pckt_seq == inbuf->khp_data_seq_num) { return i; } } } /* * If there is no buffer that's handling the current seq_num * either it's new or it's been reclaimed already. * check if it's been reclaimed/seen before using the defrag circular * buffer. If the pckt has been seen before, the buffer expired (ETIME) * and there is no point to try to defrag it again. */ if (!_seq_num_lookup(src_host, inbuf->khp_data_seq_num, 1, 0)) { errno = ETIME; return -1; } /* * register the pckt as seen */ _seq_num_set(src_host, inbuf->khp_data_seq_num, 1); /* * see if there is a free buffer */ for (i = 0; i < KNET_MAX_LINK; i++) { if (!src_host->defrag_buf[i].in_use) { return i; } } /* * at this point, there are no free buffers, the pckt is new * and we need to reclaim a buffer, and we will take the one * with the oldest timestamp. It's as good as any. */ oldest = 0; for (i = 0; i < KNET_MAX_LINK; i++) { if (timecmp(src_host->defrag_buf[i].last_update, src_host->defrag_buf[oldest].last_update) < 0) { oldest = i; } } src_host->defrag_buf[oldest].in_use = 0; return oldest; } static int pckt_defrag(knet_handle_t knet_h, struct knet_header *inbuf, ssize_t *len) { struct knet_host_defrag_buf *defrag_buf; int defrag_buf_idx; defrag_buf_idx = find_pckt_defrag_buf(knet_h, inbuf); if (defrag_buf_idx < 0) { if (errno == ETIME) { log_debug(knet_h, KNET_SUB_RX, "Defrag buffer expired"); } return 1; } defrag_buf = &knet_h->host_index[inbuf->kh_node]->defrag_buf[defrag_buf_idx]; /* * if the buf is not is use, then make sure it's clean */ if (!defrag_buf->in_use) { memset(defrag_buf, 0, sizeof(struct knet_host_defrag_buf)); defrag_buf->in_use = 1; defrag_buf->pckt_seq = inbuf->khp_data_seq_num; } /* * update timestamp on the buffer */ clock_gettime(CLOCK_MONOTONIC, &defrag_buf->last_update); /* * check if we already received this fragment */ if (defrag_buf->frag_map[inbuf->khp_data_frag_seq]) { /* * if we have received this fragment and we didn't clear the buffer * it means that we don't have all fragments yet */ return 1; } /* * we need to handle the last packet with gloves due to its different size */ if (inbuf->khp_data_frag_seq == inbuf->khp_data_frag_num) { defrag_buf->last_frag_size = *len; /* * in the event when the last packet arrives first, * we still don't know the offset vs the other fragments (based on MTU), * so we store the fragment at the end of the buffer where it's safe * and take a copy of the len so that we can restore its offset later. * remember we can't use the local MTU for this calculation because pMTU * can be asymettric between the same hosts. */ if (!defrag_buf->frag_size) { defrag_buf->last_first = 1; memmove(defrag_buf->buf + (KNET_MAX_PACKET_SIZE - *len), inbuf->khp_data_userdata, *len); } } else { defrag_buf->frag_size = *len; } memmove(defrag_buf->buf + ((inbuf->khp_data_frag_seq - 1) * defrag_buf->frag_size), inbuf->khp_data_userdata, *len); defrag_buf->frag_recv++; defrag_buf->frag_map[inbuf->khp_data_frag_seq] = 1; /* * check if we received all the fragments */ if (defrag_buf->frag_recv == inbuf->khp_data_frag_num) { /* * special case the last pckt */ if (defrag_buf->last_first) { memmove(defrag_buf->buf + ((inbuf->khp_data_frag_num - 1) * defrag_buf->frag_size), defrag_buf->buf + (KNET_MAX_PACKET_SIZE - defrag_buf->last_frag_size), defrag_buf->last_frag_size); } /* * recalculate packet lenght */ *len = ((inbuf->khp_data_frag_num - 1) * defrag_buf->frag_size) + defrag_buf->last_frag_size; /* * copy the pckt back in the user data */ memmove(inbuf->khp_data_userdata, defrag_buf->buf, *len); /* * free this buffer */ defrag_buf->in_use = 0; return 0; } return 1; } static void _parse_recv_from_links(knet_handle_t knet_h, int sockfd, const struct knet_mmsghdr *msg) { int err = 0, savederrno = 0; ssize_t outlen; struct knet_host *src_host; struct knet_link *src_link; unsigned long long latency_last; knet_node_id_t dst_host_ids[KNET_MAX_HOST]; size_t dst_host_ids_entries = 0; int bcast = 1; int was_decrypted = 0; uint64_t crypt_time = 0; struct timespec recvtime; struct knet_header *inbuf = msg->msg_hdr.msg_iov->iov_base; unsigned char *outbuf = (unsigned char *)msg->msg_hdr.msg_iov->iov_base; ssize_t len = msg->msg_len; struct knet_hostinfo *knet_hostinfo; struct iovec iov_out[1]; int8_t channel; struct sockaddr_storage pckt_src; seq_num_t recv_seq_num; int wipe_bufs = 0; if (knet_h->crypto_instance) { struct timespec start_time; struct timespec end_time; clock_gettime(CLOCK_MONOTONIC, &start_time); if (crypto_authenticate_and_decrypt(knet_h, (unsigned char *)inbuf, len, knet_h->recv_from_links_buf_decrypt, &outlen) < 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to decrypt/auth packet"); return; } clock_gettime(CLOCK_MONOTONIC, &end_time); timespec_diff(start_time, end_time, &crypt_time); if (crypt_time < knet_h->stats.rx_crypt_time_min) { knet_h->stats.rx_crypt_time_min = crypt_time; } if (crypt_time > knet_h->stats.rx_crypt_time_max) { knet_h->stats.rx_crypt_time_max = crypt_time; } len = outlen; inbuf = (struct knet_header *)knet_h->recv_from_links_buf_decrypt; was_decrypted++; } if (len < (ssize_t)(KNET_HEADER_SIZE + 1)) { log_debug(knet_h, KNET_SUB_RX, "Packet is too short: %ld", (long)len); return; } if (inbuf->kh_version != KNET_HEADER_VERSION) { log_debug(knet_h, KNET_SUB_RX, "Packet version does not match"); return; } inbuf->kh_node = ntohs(inbuf->kh_node); src_host = knet_h->host_index[inbuf->kh_node]; if (src_host == NULL) { /* host not found */ log_debug(knet_h, KNET_SUB_RX, "Unable to find source host for this packet"); return; } src_link = NULL; src_link = src_host->link + (inbuf->khp_ping_link % KNET_MAX_LINK); if ((inbuf->kh_type & KNET_HEADER_TYPE_PMSK) != 0) { if (src_link->dynamic == KNET_LINK_DYNIP) { /* * cpyaddrport will only copy address and port of the incoming * packet and strip extra bits such as flow and scopeid */ cpyaddrport(&pckt_src, msg->msg_hdr.msg_name); if (cmpaddr(&src_link->dst_addr, sockaddr_len(&src_link->dst_addr), &pckt_src, sockaddr_len(&pckt_src)) != 0) { log_debug(knet_h, KNET_SUB_RX, "host: %u link: %u appears to have changed ip address", src_host->host_id, src_link->link_id); memmove(&src_link->dst_addr, &pckt_src, sizeof(struct sockaddr_storage)); if (knet_addrtostr(&src_link->dst_addr, sockaddr_len(msg->msg_hdr.msg_name), src_link->status.dst_ipaddr, KNET_MAX_HOST_LEN, src_link->status.dst_port, KNET_MAX_PORT_LEN) != 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to resolve ???"); snprintf(src_link->status.dst_ipaddr, KNET_MAX_HOST_LEN - 1, "Unknown!!!"); snprintf(src_link->status.dst_port, KNET_MAX_PORT_LEN - 1, "??"); } else { log_info(knet_h, KNET_SUB_RX, "host: %u link: %u new connection established from: %s %s", src_host->host_id, src_link->link_id, src_link->status.dst_ipaddr, src_link->status.dst_port); } } /* * transport has already accepted the connection here * otherwise we would not be receiving packets */ transport_link_dyn_connect(knet_h, sockfd, src_link); } } switch (inbuf->kh_type) { case KNET_HEADER_TYPE_HOST_INFO: case KNET_HEADER_TYPE_DATA: /* * TODO: should we accept data even if we can't reply to the other node? * how would that work with SCTP and guaranteed delivery? */ if (!src_host->status.reachable) { log_debug(knet_h, KNET_SUB_RX, "Source host %u not reachable yet", src_host->host_id); //return; } inbuf->khp_data_seq_num = ntohs(inbuf->khp_data_seq_num); channel = inbuf->khp_data_channel; src_host->got_data = 1; if (src_link) { src_link->status.stats.rx_data_packets++; src_link->status.stats.rx_data_bytes += len; } if (!_seq_num_lookup(src_host, inbuf->khp_data_seq_num, 0, 0)) { if (src_host->link_handler_policy != KNET_LINK_POLICY_ACTIVE) { log_debug(knet_h, KNET_SUB_RX, "Packet has already been delivered"); } return; } if (inbuf->khp_data_frag_num > 1) { /* * len as received from the socket also includes extra stuff * that the defrag code doesn't care about. So strip it * here and readd only for repadding once we are done * defragging */ len = len - KNET_HEADER_DATA_SIZE; if (pckt_defrag(knet_h, inbuf, &len)) { return; } len = len + KNET_HEADER_DATA_SIZE; } if (inbuf->khp_data_compress) { ssize_t decmp_outlen = KNET_DATABUFSIZE_COMPRESS; struct timespec start_time; struct timespec end_time; uint64_t compress_time; clock_gettime(CLOCK_MONOTONIC, &start_time); err = decompress(knet_h, inbuf->khp_data_compress, (const unsigned char *)inbuf->khp_data_userdata, len - KNET_HEADER_DATA_SIZE, knet_h->recv_from_links_buf_decompress, &decmp_outlen); if (!err) { /* Collect stats */ clock_gettime(CLOCK_MONOTONIC, &end_time); timespec_diff(start_time, end_time, &compress_time); if (compress_time < knet_h->stats.rx_compress_time_min) { knet_h->stats.rx_compress_time_min = compress_time; } if (compress_time > knet_h->stats.rx_compress_time_max) { knet_h->stats.rx_compress_time_max = compress_time; } knet_h->stats.rx_compress_time_ave = (knet_h->stats.rx_compress_time_ave * knet_h->stats.rx_compressed_packets + compress_time) / (knet_h->stats.rx_compressed_packets+1); knet_h->stats.rx_compressed_packets++; knet_h->stats.rx_compressed_original_bytes += decmp_outlen; knet_h->stats.rx_compressed_size_bytes += len - KNET_HEADER_SIZE; memmove(inbuf->khp_data_userdata, knet_h->recv_from_links_buf_decompress, decmp_outlen); len = decmp_outlen + KNET_HEADER_DATA_SIZE; } else { log_warn(knet_h, KNET_SUB_COMPRESS, "Unable to decompress packet (%d): %s", err, strerror(errno)); return; } } if (inbuf->kh_type == KNET_HEADER_TYPE_DATA) { if (knet_h->enabled != 1) /* data forward is disabled */ break; /* Only update the crypto overhead for data packets. Mainly to be consistent with TX */ knet_h->stats.rx_crypt_time_ave = (knet_h->stats.rx_crypt_time_ave * knet_h->stats.rx_crypt_packets + crypt_time) / (knet_h->stats.rx_crypt_packets+1); knet_h->stats.rx_crypt_packets++; if (knet_h->dst_host_filter_fn) { size_t host_idx; int found = 0; bcast = knet_h->dst_host_filter_fn( knet_h->dst_host_filter_fn_private_data, (const unsigned char *)inbuf->khp_data_userdata, len - KNET_HEADER_DATA_SIZE, KNET_NOTIFY_RX, knet_h->host_id, inbuf->kh_node, &channel, dst_host_ids, &dst_host_ids_entries); if (bcast < 0) { log_debug(knet_h, KNET_SUB_RX, "Error from dst_host_filter_fn: %d", bcast); return; } if ((!bcast) && (!dst_host_ids_entries)) { log_debug(knet_h, KNET_SUB_RX, "Message is unicast but no dst_host_ids_entries"); return; } /* check if we are dst for this packet */ if (!bcast) { + if (dst_host_ids_entries > KNET_MAX_HOST) { + log_debug(knet_h, KNET_SUB_RX, "dst_host_filter_fn returned too many destinations"); + return; + } for (host_idx = 0; host_idx < dst_host_ids_entries; host_idx++) { if (dst_host_ids[host_idx] == knet_h->host_id) { found = 1; break; } } if (!found) { log_debug(knet_h, KNET_SUB_RX, "Packet is not for us"); return; } } } } if (inbuf->kh_type == KNET_HEADER_TYPE_DATA) { if (!knet_h->sockfd[channel].in_use) { log_debug(knet_h, KNET_SUB_RX, "received packet for channel %d but there is no local sock connected", channel); return; } memset(iov_out, 0, sizeof(iov_out)); iov_out[0].iov_base = (void *) inbuf->khp_data_userdata; iov_out[0].iov_len = len - KNET_HEADER_DATA_SIZE; outlen = writev(knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created], iov_out, 1); if (outlen <= 0) { knet_h->sock_notify_fn(knet_h->sock_notify_fn_private_data, knet_h->sockfd[channel].sockfd[0], channel, KNET_NOTIFY_RX, outlen, errno); return; } if ((size_t)outlen == iov_out[0].iov_len) { _seq_num_set(src_host, inbuf->khp_data_seq_num, 0); } } else { /* HOSTINFO */ knet_hostinfo = (struct knet_hostinfo *)inbuf->khp_data_userdata; if (knet_hostinfo->khi_bcast == KNET_HOSTINFO_UCAST) { bcast = 0; knet_hostinfo->khi_dst_node_id = ntohs(knet_hostinfo->khi_dst_node_id); } if (!_seq_num_lookup(src_host, inbuf->khp_data_seq_num, 0, 0)) { return; } _seq_num_set(src_host, inbuf->khp_data_seq_num, 0); switch(knet_hostinfo->khi_type) { case KNET_HOSTINFO_TYPE_LINK_UP_DOWN: break; case KNET_HOSTINFO_TYPE_LINK_TABLE: break; default: log_warn(knet_h, KNET_SUB_RX, "Receiving unknown host info message from host %u", src_host->host_id); break; } } break; case KNET_HEADER_TYPE_PING: outlen = KNET_HEADER_PING_SIZE; inbuf->kh_type = KNET_HEADER_TYPE_PONG; inbuf->kh_node = htons(knet_h->host_id); recv_seq_num = ntohs(inbuf->khp_ping_seq_num); src_link->status.stats.rx_ping_packets++; src_link->status.stats.rx_ping_bytes += len; wipe_bufs = 0; if (!inbuf->khp_ping_timed) { /* * we might be receiving this message from all links, but we want * to process it only the first time */ if (recv_seq_num != src_host->untimed_rx_seq_num) { /* * cache the untimed seq num */ src_host->untimed_rx_seq_num = recv_seq_num; /* * if the host has received data in between * untimed ping, then we don't need to wipe the bufs */ if (src_host->got_data) { src_host->got_data = 0; wipe_bufs = 0; } else { wipe_bufs = 1; } } _seq_num_lookup(src_host, recv_seq_num, 0, wipe_bufs); } else { /* * pings always arrives in bursts over all the link * catch the first of them to cache the seq num and * avoid duplicate processing */ if (recv_seq_num != src_host->timed_rx_seq_num) { src_host->timed_rx_seq_num = recv_seq_num; if (recv_seq_num == 0) { _seq_num_lookup(src_host, recv_seq_num, 0, 1); } } } if (knet_h->crypto_instance) { if (crypto_encrypt_and_sign(knet_h, (const unsigned char *)inbuf, len, knet_h->recv_from_links_buf_crypt, &outlen) < 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to encrypt pong packet"); break; } outbuf = knet_h->recv_from_links_buf_crypt; knet_h->stats_extra.tx_crypt_pong_packets++; } retry_pong: len = sendto(src_link->outsock, outbuf, outlen, MSG_DONTWAIT | MSG_NOSIGNAL, (struct sockaddr *) &src_link->dst_addr, sizeof(struct sockaddr_storage)); savederrno = errno; if (len != outlen) { err = transport_tx_sock_error(knet_h, src_link->transport_type, src_link->outsock, len, savederrno); switch(err) { case -1: /* unrecoverable error */ log_debug(knet_h, KNET_SUB_RX, "Unable to send pong reply (sock: %d) packet (sendto): %d %s. recorded src ip: %s src port: %s dst ip: %s dst port: %s", src_link->outsock, errno, strerror(errno), src_link->status.src_ipaddr, src_link->status.src_port, src_link->status.dst_ipaddr, src_link->status.dst_port); src_link->status.stats.tx_pong_errors++; break; case 0: /* ignore error and continue */ break; case 1: /* retry to send those same data */ src_link->status.stats.tx_pong_retries++; goto retry_pong; break; } } src_link->status.stats.tx_pong_packets++; src_link->status.stats.tx_pong_bytes += outlen; break; case KNET_HEADER_TYPE_PONG: src_link->status.stats.rx_pong_packets++; src_link->status.stats.rx_pong_bytes += len; clock_gettime(CLOCK_MONOTONIC, &src_link->status.pong_last); memmove(&recvtime, &inbuf->khp_ping_time[0], sizeof(struct timespec)); timespec_diff(recvtime, src_link->status.pong_last, &latency_last); src_link->status.latency = ((src_link->status.latency * src_link->latency_exp) + ((latency_last / 1000llu) * (src_link->latency_fix - src_link->latency_exp))) / src_link->latency_fix; if (src_link->status.latency < src_link->pong_timeout_adj) { if (!src_link->status.connected) { if (src_link->received_pong >= src_link->pong_count) { log_info(knet_h, KNET_SUB_RX, "host: %u link: %u is up", src_host->host_id, src_link->link_id); _link_updown(knet_h, src_host->host_id, src_link->link_id, src_link->status.enabled, 1); } else { src_link->received_pong++; log_debug(knet_h, KNET_SUB_RX, "host: %u link: %u received pong: %u", src_host->host_id, src_link->link_id, src_link->received_pong); } } } /* Calculate latency stats */ if (src_link->status.latency > src_link->status.stats.latency_max) { src_link->status.stats.latency_max = src_link->status.latency; } if (src_link->status.latency < src_link->status.stats.latency_min) { src_link->status.stats.latency_min = src_link->status.latency; } src_link->status.stats.latency_ave = (src_link->status.stats.latency_ave * src_link->status.stats.latency_samples + src_link->status.latency) / (src_link->status.stats.latency_samples+1); src_link->status.stats.latency_samples++; break; case KNET_HEADER_TYPE_PMTUD: src_link->status.stats.rx_pmtu_packets++; src_link->status.stats.rx_pmtu_bytes += len; outlen = KNET_HEADER_PMTUD_SIZE; inbuf->kh_type = KNET_HEADER_TYPE_PMTUD_REPLY; inbuf->kh_node = htons(knet_h->host_id); if (knet_h->crypto_instance) { if (crypto_encrypt_and_sign(knet_h, (const unsigned char *)inbuf, len, knet_h->recv_from_links_buf_crypt, &outlen) < 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to encrypt PMTUd reply packet"); break; } outbuf = knet_h->recv_from_links_buf_crypt; knet_h->stats_extra.tx_crypt_pmtu_reply_packets++; } savederrno = pthread_mutex_lock(&knet_h->tx_mutex); if (savederrno) { log_err(knet_h, KNET_SUB_RX, "Unable to get TX mutex lock: %s", strerror(savederrno)); goto out_pmtud; } retry_pmtud: len = sendto(src_link->outsock, outbuf, outlen, MSG_DONTWAIT | MSG_NOSIGNAL, (struct sockaddr *) &src_link->dst_addr, sizeof(struct sockaddr_storage)); savederrno = errno; if (len != outlen) { err = transport_tx_sock_error(knet_h, src_link->transport_type, src_link->outsock, len, savederrno); switch(err) { case -1: /* unrecoverable error */ log_debug(knet_h, KNET_SUB_RX, "Unable to send PMTUd reply (sock: %d) packet (sendto): %d %s. recorded src ip: %s src port: %s dst ip: %s dst port: %s", src_link->outsock, errno, strerror(errno), src_link->status.src_ipaddr, src_link->status.src_port, src_link->status.dst_ipaddr, src_link->status.dst_port); src_link->status.stats.tx_pmtu_errors++; break; case 0: /* ignore error and continue */ src_link->status.stats.tx_pmtu_errors++; break; case 1: /* retry to send those same data */ src_link->status.stats.tx_pmtu_retries++; goto retry_pmtud; break; } } pthread_mutex_unlock(&knet_h->tx_mutex); out_pmtud: break; case KNET_HEADER_TYPE_PMTUD_REPLY: src_link->status.stats.rx_pmtu_packets++; src_link->status.stats.rx_pmtu_bytes += len; if (pthread_mutex_lock(&knet_h->pmtud_mutex) != 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to get mutex lock"); break; } src_link->last_recv_mtu = inbuf->khp_pmtud_size; pthread_cond_signal(&knet_h->pmtud_cond); pthread_mutex_unlock(&knet_h->pmtud_mutex); break; default: return; } } static void _handle_recv_from_links(knet_handle_t knet_h, int sockfd, struct knet_mmsghdr *msg) { int err, savederrno; int i, msg_recv, transport; if (pthread_rwlock_rdlock(&knet_h->global_rwlock) != 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to get global read lock"); return; } if (_is_valid_fd(knet_h, sockfd) < 1) { /* * this is normal if a fd got an event and before we grab the read lock * and the link is removed by another thread */ goto exit_unlock; } transport = knet_h->knet_transport_fd_tracker[sockfd].transport; /* * reset msg_namelen to buffer size because after recvmmsg * each msg_namelen will contain sizeof sockaddr_in or sockaddr_in6 */ for (i = 0; i < PCKT_RX_BUFS; i++) { msg[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_storage); } msg_recv = _recvmmsg(sockfd, &msg[0], PCKT_RX_BUFS, MSG_DONTWAIT | MSG_NOSIGNAL); savederrno = errno; /* * WARNING: man page for recvmmsg is wrong. Kernel implementation here: * recvmmsg can return: * -1 on error * 0 if the previous run of recvmmsg recorded an error on the socket * N number of messages (see exception below). * * If there is an error from recvmsg after receiving a frame or more, the recvmmsg * loop is interrupted, error recorded in the socket (getsockopt(SO_ERROR) and * it will be visibile in the next run. * * Need to be careful how we handle errors at this stage. * * error messages need to be handled on a per transport/protocol base * at this point we have different layers of error handling * - msg_recv < 0 -> error from this run * msg_recv = 0 -> error from previous run and error on socket needs to be cleared * - per-transport message data * example: msg[i].msg_hdr.msg_flags & MSG_NOTIFICATION or msg_len for SCTP == EOF, * but for UDP it is perfectly legal to receive a 0 bytes message.. go figure * - NOTE: on SCTP MSG_NOTIFICATION we get msg_recv == PCKT_FRAG_MAX messages and no * errno set. That means the error api needs to be able to abort the loop below. */ if (msg_recv <= 0) { transport_rx_sock_error(knet_h, transport, sockfd, msg_recv, savederrno); goto exit_unlock; } for (i = 0; i < msg_recv; i++) { err = transport_rx_is_data(knet_h, transport, sockfd, &msg[i]); /* * TODO: make this section silent once we are confident * all protocols packet handlers are good */ switch(err) { case -1: /* on error */ log_debug(knet_h, KNET_SUB_RX, "Transport reported error parsing packet"); goto exit_unlock; break; case 0: /* packet is not data and we should continue the packet process loop */ log_debug(knet_h, KNET_SUB_RX, "Transport reported no data, continue"); break; case 1: /* packet is not data and we should STOP the packet process loop */ log_debug(knet_h, KNET_SUB_RX, "Transport reported no data, stop"); goto exit_unlock; break; case 2: /* packet is data and should be parsed as such */ _parse_recv_from_links(knet_h, sockfd, &msg[i]); break; } } exit_unlock: pthread_rwlock_unlock(&knet_h->global_rwlock); } void *_handle_recv_from_links_thread(void *data) { int i, nev; knet_handle_t knet_h = (knet_handle_t) data; struct epoll_event events[KNET_EPOLL_MAX_EVENTS]; struct sockaddr_storage address[PCKT_RX_BUFS]; struct knet_mmsghdr msg[PCKT_RX_BUFS]; struct iovec iov_in[PCKT_RX_BUFS]; memset(&msg, 0, sizeof(msg)); for (i = 0; i < PCKT_RX_BUFS; i++) { iov_in[i].iov_base = (void *)knet_h->recv_from_links_buf[i]; iov_in[i].iov_len = KNET_DATABUFSIZE; memset(&msg[i].msg_hdr, 0, sizeof(struct msghdr)); msg[i].msg_hdr.msg_name = &address[i]; msg[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_storage); msg[i].msg_hdr.msg_iov = &iov_in[i]; msg[i].msg_hdr.msg_iovlen = 1; } while (!shutdown_in_progress(knet_h)) { nev = epoll_wait(knet_h->recv_from_links_epollfd, events, KNET_EPOLL_MAX_EVENTS, -1); for (i = 0; i < nev; i++) { _handle_recv_from_links(knet_h, events[i].data.fd, msg); } } return NULL; } diff --git a/libknet/threads_tx.c b/libknet/threads_tx.c index 5d4062ef..3c7a7f3b 100644 --- a/libknet/threads_tx.c +++ b/libknet/threads_tx.c @@ -1,737 +1,745 @@ /* * Copyright (C) 2012-2018 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under GPL-2.0+, LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include "compat.h" #include "compress.h" #include "crypto.h" #include "host.h" #include "link.h" #include "logging.h" #include "transports.h" #include "transport_common.h" #include "threads_common.h" #include "threads_heartbeat.h" #include "threads_tx.h" #include "netutils.h" /* * SEND */ static int _dispatch_to_links(knet_handle_t knet_h, struct knet_host *dst_host, struct knet_mmsghdr *msg, int msgs_to_send) { int link_idx, msg_idx, sent_msgs, prev_sent, progress; int err = 0, savederrno = 0; unsigned int i; struct knet_mmsghdr *cur; struct knet_link *cur_link; for (link_idx = 0; link_idx < dst_host->active_link_entries; link_idx++) { sent_msgs = 0; prev_sent = 0; progress = 1; cur_link = &dst_host->link[dst_host->active_links[link_idx]]; if (cur_link->transport_type == KNET_TRANSPORT_LOOPBACK) { continue; } msg_idx = 0; while (msg_idx < msgs_to_send) { msg[msg_idx].msg_hdr.msg_name = &cur_link->dst_addr; /* Cast for Linux/BSD compatibility */ for (i=0; i<(unsigned int)msg[msg_idx].msg_hdr.msg_iovlen; i++) { cur_link->status.stats.tx_data_bytes += msg[msg_idx].msg_hdr.msg_iov[i].iov_len; } cur_link->status.stats.tx_data_packets++; msg_idx++; } retry: cur = &msg[prev_sent]; sent_msgs = _sendmmsg(dst_host->link[dst_host->active_links[link_idx]].outsock, &cur[0], msgs_to_send - prev_sent, MSG_DONTWAIT | MSG_NOSIGNAL); savederrno = errno; err = transport_tx_sock_error(knet_h, dst_host->link[dst_host->active_links[link_idx]].transport_type, dst_host->link[dst_host->active_links[link_idx]].outsock, sent_msgs, savederrno); switch(err) { case -1: /* unrecoverable error */ cur_link->status.stats.tx_data_errors++; goto out_unlock; break; case 0: /* ignore error and continue */ break; case 1: /* retry to send those same data */ cur_link->status.stats.tx_data_retries++; goto retry; break; } prev_sent = prev_sent + sent_msgs; if ((sent_msgs >= 0) && (prev_sent < msgs_to_send)) { if ((sent_msgs) || (progress)) { if (sent_msgs) { progress = 1; } else { progress = 0; } #ifdef DEBUG log_debug(knet_h, KNET_SUB_TX, "Unable to send all (%d/%d) data packets to host %s (%u) link %s:%s (%u)", sent_msgs, msg_idx, dst_host->name, dst_host->host_id, dst_host->link[dst_host->active_links[link_idx]].status.dst_ipaddr, dst_host->link[dst_host->active_links[link_idx]].status.dst_port, dst_host->link[dst_host->active_links[link_idx]].link_id); #endif goto retry; } if (!progress) { savederrno = EAGAIN; err = -1; goto out_unlock; } } if ((dst_host->link_handler_policy == KNET_LINK_POLICY_RR) && (dst_host->active_link_entries > 1)) { uint8_t cur_link_id = dst_host->active_links[0]; memmove(&dst_host->active_links[0], &dst_host->active_links[1], KNET_MAX_LINK - 1); dst_host->active_links[dst_host->active_link_entries - 1] = cur_link_id; break; } } out_unlock: errno = savederrno; return err; } static int _parse_recv_from_sock(knet_handle_t knet_h, size_t inlen, int8_t channel, int is_sync) { size_t outlen, frag_len; struct knet_host *dst_host; knet_node_id_t dst_host_ids_temp[KNET_MAX_HOST]; size_t dst_host_ids_entries_temp = 0; knet_node_id_t dst_host_ids[KNET_MAX_HOST]; size_t dst_host_ids_entries = 0; int bcast = 1; struct knet_hostinfo *knet_hostinfo; struct iovec iov_out[PCKT_FRAG_MAX][2]; int iovcnt_out = 2; uint8_t frag_idx; unsigned int temp_data_mtu; size_t host_idx; int send_mcast = 0; struct knet_header *inbuf; int savederrno = 0; int err = 0; seq_num_t tx_seq_num; struct knet_mmsghdr msg[PCKT_FRAG_MAX]; int msgs_to_send, msg_idx; unsigned int i; int j; int send_local = 0; int data_compressed = 0; size_t uncrypted_frag_size; inbuf = knet_h->recv_from_sock_buf; if ((knet_h->enabled != 1) && (inbuf->kh_type != KNET_HEADER_TYPE_HOST_INFO)) { /* data forward is disabled */ log_debug(knet_h, KNET_SUB_TX, "Received data packet but forwarding is disabled"); savederrno = ECANCELED; err = -1; goto out_unlock; } /* * move this into a separate function to expand on * extra switching rules */ switch(inbuf->kh_type) { case KNET_HEADER_TYPE_DATA: if (knet_h->dst_host_filter_fn) { bcast = knet_h->dst_host_filter_fn( knet_h->dst_host_filter_fn_private_data, (const unsigned char *)inbuf->khp_data_userdata, inlen, KNET_NOTIFY_TX, knet_h->host_id, knet_h->host_id, &channel, dst_host_ids_temp, &dst_host_ids_entries_temp); if (bcast < 0) { log_debug(knet_h, KNET_SUB_TX, "Error from dst_host_filter_fn: %d", bcast); savederrno = EFAULT; err = -1; goto out_unlock; } if ((!bcast) && (!dst_host_ids_entries_temp)) { log_debug(knet_h, KNET_SUB_TX, "Message is unicast but no dst_host_ids_entries"); savederrno = EINVAL; err = -1; goto out_unlock; } + + if ((!bcast) && + (dst_host_ids_entries_temp > KNET_MAX_HOST)) { + log_debug(knet_h, KNET_SUB_TX, "dst_host_filter_fn returned too many destinations"); + savederrno = EINVAL; + err = -1; + goto out_unlock; + } } /* Send to localhost if appropriate and enabled */ if (knet_h->has_loop_link) { send_local = 0; if (bcast) { send_local = 1; } else { for (i=0; i< dst_host_ids_entries_temp; i++) { if (dst_host_ids_temp[i] == knet_h->host_id) { send_local = 1; } } } if (send_local) { const unsigned char *buf = inbuf->khp_data_userdata; ssize_t buflen = inlen; struct knet_link *local_link; local_link = knet_h->host_index[knet_h->host_id]->link; local_retry: err = write(knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created], buf, buflen); if (err < 0) { log_err(knet_h, KNET_SUB_TRANSP_LOOPBACK, "send local failed. error=%s\n", strerror(errno)); local_link->status.stats.tx_data_errors++; } if (err > 0 && err < buflen) { log_debug(knet_h, KNET_SUB_TRANSP_LOOPBACK, "send local incomplete=%d bytes of %zu\n", err, inlen); local_link->status.stats.tx_data_retries++; buf += err; buflen -= err; usleep(KNET_THREADS_TIMERES / 16); goto local_retry; } if (err == buflen) { local_link->status.stats.tx_data_packets++; local_link->status.stats.tx_data_bytes += inlen; } } } break; case KNET_HEADER_TYPE_HOST_INFO: knet_hostinfo = (struct knet_hostinfo *)inbuf->khp_data_userdata; if (knet_hostinfo->khi_bcast == KNET_HOSTINFO_UCAST) { bcast = 0; dst_host_ids_temp[0] = knet_hostinfo->khi_dst_node_id; dst_host_ids_entries_temp = 1; knet_hostinfo->khi_dst_node_id = htons(knet_hostinfo->khi_dst_node_id); } break; default: log_warn(knet_h, KNET_SUB_TX, "Receiving unknown messages from socket"); savederrno = ENOMSG; err = -1; goto out_unlock; break; } if (is_sync) { if ((bcast) || ((!bcast) && (dst_host_ids_entries_temp > 1))) { log_debug(knet_h, KNET_SUB_TX, "knet_send_sync is only supported with unicast packets for one destination"); savederrno = E2BIG; err = -1; goto out_unlock; } } /* * check destinations hosts before spending time * in fragmenting/encrypting packets to save * time processing data for unreachable hosts. * for unicast, also remap the destination data * to skip unreachable hosts. */ if (!bcast) { dst_host_ids_entries = 0; for (host_idx = 0; host_idx < dst_host_ids_entries_temp; host_idx++) { dst_host = knet_h->host_index[dst_host_ids_temp[host_idx]]; if (!dst_host) { continue; } if (!(dst_host->host_id == knet_h->host_id && knet_h->has_loop_link) && dst_host->status.reachable) { dst_host_ids[dst_host_ids_entries] = dst_host_ids_temp[host_idx]; dst_host_ids_entries++; } } if (!dst_host_ids_entries) { savederrno = EHOSTDOWN; err = -1; goto out_unlock; } } else { send_mcast = 0; for (dst_host = knet_h->host_head; dst_host != NULL; dst_host = dst_host->next) { if (!(dst_host->host_id == knet_h->host_id && knet_h->has_loop_link) && dst_host->status.reachable) { send_mcast = 1; break; } } if (!send_mcast) { savederrno = EHOSTDOWN; err = -1; goto out_unlock; } } if (!knet_h->data_mtu) { /* * using MIN_MTU_V4 for data mtu is not completely accurate but safe enough */ log_debug(knet_h, KNET_SUB_TX, "Received data packet but data MTU is still unknown." " Packet might not be delivered." " Assuming minimum IPv4 MTU (%d)", KNET_PMTUD_MIN_MTU_V4); temp_data_mtu = KNET_PMTUD_MIN_MTU_V4; } else { /* * take a copy of the mtu to avoid value changing under * our feet while we are sending a fragmented pckt */ temp_data_mtu = knet_h->data_mtu; } /* * compress data */ if ((knet_h->compress_model > 0) && (inlen > knet_h->compress_threshold)) { size_t cmp_outlen = KNET_DATABUFSIZE_COMPRESS; struct timespec start_time; struct timespec end_time; uint64_t compress_time; clock_gettime(CLOCK_MONOTONIC, &start_time); err = compress(knet_h, (const unsigned char *)inbuf->khp_data_userdata, inlen, knet_h->send_to_links_buf_compress, (ssize_t *)&cmp_outlen); if (err < 0) { log_warn(knet_h, KNET_SUB_COMPRESS, "Compression failed (%d): %s", err, strerror(errno)); } else { /* Collect stats */ clock_gettime(CLOCK_MONOTONIC, &end_time); timespec_diff(start_time, end_time, &compress_time); if (compress_time < knet_h->stats.tx_compress_time_min) { knet_h->stats.tx_compress_time_min = compress_time; } if (compress_time > knet_h->stats.tx_compress_time_max) { knet_h->stats.tx_compress_time_max = compress_time; } knet_h->stats.tx_compress_time_ave = (unsigned long long)(knet_h->stats.tx_compress_time_ave * knet_h->stats.tx_compressed_packets + compress_time) / (knet_h->stats.tx_compressed_packets+1); knet_h->stats.tx_compressed_packets++; knet_h->stats.tx_compressed_original_bytes += inlen; knet_h->stats.tx_compressed_size_bytes += cmp_outlen; if (cmp_outlen < inlen) { memmove(inbuf->khp_data_userdata, knet_h->send_to_links_buf_compress, cmp_outlen); inlen = cmp_outlen; data_compressed = 1; } } } if ((knet_h->compress_model > 0) && (inlen <= knet_h->compress_threshold)) { knet_h->stats.tx_uncompressed_packets++; } /* * prepare the outgoing buffers */ frag_len = inlen; frag_idx = 0; inbuf->khp_data_bcast = bcast; inbuf->khp_data_frag_num = ceil((float)inlen / temp_data_mtu); inbuf->khp_data_channel = channel; if (data_compressed) { inbuf->khp_data_compress = knet_h->compress_model; } else { inbuf->khp_data_compress = 0; } if (pthread_mutex_lock(&knet_h->tx_seq_num_mutex)) { log_debug(knet_h, KNET_SUB_TX, "Unable to get seq mutex lock"); goto out_unlock; } knet_h->tx_seq_num++; /* * force seq_num 0 to detect a node that has crashed and rejoining * the knet instance. seq_num 0 will clear the buffers in the RX * thread */ if (knet_h->tx_seq_num == 0) { knet_h->tx_seq_num++; } /* * cache the value in locked context */ tx_seq_num = knet_h->tx_seq_num; inbuf->khp_data_seq_num = htons(knet_h->tx_seq_num); pthread_mutex_unlock(&knet_h->tx_seq_num_mutex); /* * forcefully broadcast a ping to all nodes every SEQ_MAX / 8 * pckts. * this solves 2 problems: * 1) on TX socket overloads we generate extra pings to keep links alive * 2) in 3+ nodes setup, where all the traffic is flowing between node 1 and 2, * node 3+ will be able to keep in sync on the TX seq_num even without * receiving traffic or pings in betweens. This avoids issues with * rollover of the circular buffer */ if (tx_seq_num % (SEQ_MAX / 8) == 0) { _send_pings(knet_h, 0); } if (inbuf->khp_data_frag_num > 1) { while (frag_idx < inbuf->khp_data_frag_num) { /* * set the iov_base */ iov_out[frag_idx][0].iov_base = (void *)knet_h->send_to_links_buf[frag_idx]; iov_out[frag_idx][0].iov_len = KNET_HEADER_DATA_SIZE; iov_out[frag_idx][1].iov_base = inbuf->khp_data_userdata + (temp_data_mtu * frag_idx); /* * set the len */ if (frag_len > temp_data_mtu) { iov_out[frag_idx][1].iov_len = temp_data_mtu; } else { iov_out[frag_idx][1].iov_len = frag_len; } /* * copy the frag info on all buffers */ knet_h->send_to_links_buf[frag_idx]->kh_type = inbuf->kh_type; knet_h->send_to_links_buf[frag_idx]->khp_data_seq_num = inbuf->khp_data_seq_num; knet_h->send_to_links_buf[frag_idx]->khp_data_frag_num = inbuf->khp_data_frag_num; knet_h->send_to_links_buf[frag_idx]->khp_data_bcast = inbuf->khp_data_bcast; knet_h->send_to_links_buf[frag_idx]->khp_data_channel = inbuf->khp_data_channel; knet_h->send_to_links_buf[frag_idx]->khp_data_compress = inbuf->khp_data_compress; frag_len = frag_len - temp_data_mtu; frag_idx++; } iovcnt_out = 2; } else { iov_out[frag_idx][0].iov_base = (void *)inbuf; iov_out[frag_idx][0].iov_len = frag_len + KNET_HEADER_DATA_SIZE; iovcnt_out = 1; } if (knet_h->crypto_instance) { struct timespec start_time; struct timespec end_time; uint64_t crypt_time; frag_idx = 0; while (frag_idx < inbuf->khp_data_frag_num) { clock_gettime(CLOCK_MONOTONIC, &start_time); if (crypto_encrypt_and_signv( knet_h, iov_out[frag_idx], iovcnt_out, knet_h->send_to_links_buf_crypt[frag_idx], (ssize_t *)&outlen) < 0) { log_debug(knet_h, KNET_SUB_TX, "Unable to encrypt packet"); savederrno = ECHILD; err = -1; goto out_unlock; } clock_gettime(CLOCK_MONOTONIC, &end_time); timespec_diff(start_time, end_time, &crypt_time); if (crypt_time < knet_h->stats.tx_crypt_time_min) { knet_h->stats.tx_crypt_time_min = crypt_time; } if (crypt_time > knet_h->stats.tx_crypt_time_max) { knet_h->stats.tx_crypt_time_max = crypt_time; } knet_h->stats.tx_crypt_time_ave = (knet_h->stats.tx_crypt_time_ave * knet_h->stats.tx_crypt_packets + crypt_time) / (knet_h->stats.tx_crypt_packets+1); uncrypted_frag_size = 0; for (j=0; j < iovcnt_out; j++) { uncrypted_frag_size += iov_out[frag_idx][j].iov_len; } knet_h->stats.tx_crypt_byte_overhead += (outlen - uncrypted_frag_size); knet_h->stats.tx_crypt_packets++; iov_out[frag_idx][0].iov_base = knet_h->send_to_links_buf_crypt[frag_idx]; iov_out[frag_idx][0].iov_len = outlen; frag_idx++; } iovcnt_out = 1; } memset(&msg, 0, sizeof(msg)); msgs_to_send = inbuf->khp_data_frag_num; msg_idx = 0; while (msg_idx < msgs_to_send) { msg[msg_idx].msg_hdr.msg_namelen = sizeof(struct sockaddr_storage); msg[msg_idx].msg_hdr.msg_iov = &iov_out[msg_idx][0]; msg[msg_idx].msg_hdr.msg_iovlen = iovcnt_out; msg_idx++; } if (!bcast) { for (host_idx = 0; host_idx < dst_host_ids_entries; host_idx++) { dst_host = knet_h->host_index[dst_host_ids[host_idx]]; err = _dispatch_to_links(knet_h, dst_host, &msg[0], msgs_to_send); savederrno = errno; if (err) { goto out_unlock; } } } else { for (dst_host = knet_h->host_head; dst_host != NULL; dst_host = dst_host->next) { if (dst_host->status.reachable) { err = _dispatch_to_links(knet_h, dst_host, &msg[0], msgs_to_send); savederrno = errno; if (err) { goto out_unlock; } } } } out_unlock: errno = savederrno; return err; } int knet_send_sync(knet_handle_t knet_h, const char *buff, const size_t buff_len, const int8_t channel) { int savederrno = 0, err = 0; if (!knet_h) { errno = EINVAL; return -1; } if (buff == NULL) { errno = EINVAL; return -1; } if (buff_len <= 0) { errno = EINVAL; return -1; } if (buff_len > KNET_MAX_PACKET_SIZE) { errno = EINVAL; return -1; } if (channel < 0) { errno = EINVAL; return -1; } if (channel >= KNET_DATAFD_MAX) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_TX, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (!knet_h->sockfd[channel].in_use) { savederrno = EINVAL; err = -1; goto out; } savederrno = pthread_mutex_lock(&knet_h->tx_mutex); if (savederrno) { log_err(knet_h, KNET_SUB_TX, "Unable to get TX mutex lock: %s", strerror(savederrno)); err = -1; goto out; } knet_h->recv_from_sock_buf->kh_type = KNET_HEADER_TYPE_DATA; memmove(knet_h->recv_from_sock_buf->khp_data_userdata, buff, buff_len); err = _parse_recv_from_sock(knet_h, buff_len, channel, 1); savederrno = errno; pthread_mutex_unlock(&knet_h->tx_mutex); out: pthread_rwlock_unlock(&knet_h->global_rwlock); errno = savederrno; return err; } static void _handle_send_to_links(knet_handle_t knet_h, struct msghdr *msg, int sockfd, int8_t channel, int type) { ssize_t inlen = 0; int savederrno = 0, docallback = 0; if ((channel >= 0) && (channel < KNET_DATAFD_MAX) && (!knet_h->sockfd[channel].is_socket)) { inlen = readv(sockfd, msg->msg_iov, 1); } else { inlen = recvmsg(sockfd, msg, MSG_DONTWAIT | MSG_NOSIGNAL); } if (inlen == 0) { savederrno = 0; docallback = 1; goto out; } if (inlen < 0) { savederrno = errno; docallback = 1; goto out; } knet_h->recv_from_sock_buf->kh_type = type; _parse_recv_from_sock(knet_h, inlen, channel, 0); out: if (inlen < 0) { struct epoll_event ev; memset(&ev, 0, sizeof(struct epoll_event)); if (epoll_ctl(knet_h->send_to_links_epollfd, EPOLL_CTL_DEL, knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created], &ev)) { log_err(knet_h, KNET_SUB_TX, "Unable to del datafd %d from linkfd epoll pool: %s", knet_h->sockfd[channel].sockfd[0], strerror(savederrno)); } else { knet_h->sockfd[channel].has_error = 1; } } if (docallback) { knet_h->sock_notify_fn(knet_h->sock_notify_fn_private_data, knet_h->sockfd[channel].sockfd[0], channel, KNET_NOTIFY_TX, inlen, savederrno); } } void *_handle_send_to_links_thread(void *data) { knet_handle_t knet_h = (knet_handle_t) data; struct epoll_event events[KNET_EPOLL_MAX_EVENTS]; int i, nev, type; int8_t channel; struct iovec iov_in; struct msghdr msg; struct sockaddr_storage address; memset(&iov_in, 0, sizeof(iov_in)); iov_in.iov_base = (void *)knet_h->recv_from_sock_buf->khp_data_userdata; iov_in.iov_len = KNET_MAX_PACKET_SIZE; memset(&msg, 0, sizeof(struct msghdr)); msg.msg_name = &address; msg.msg_namelen = sizeof(struct sockaddr_storage); msg.msg_iov = &iov_in; msg.msg_iovlen = 1; knet_h->recv_from_sock_buf->kh_version = KNET_HEADER_VERSION; knet_h->recv_from_sock_buf->khp_data_frag_seq = 0; knet_h->recv_from_sock_buf->kh_node = htons(knet_h->host_id); for (i = 0; i < PCKT_FRAG_MAX; i++) { knet_h->send_to_links_buf[i]->kh_version = KNET_HEADER_VERSION; knet_h->send_to_links_buf[i]->khp_data_frag_seq = i + 1; knet_h->send_to_links_buf[i]->kh_node = htons(knet_h->host_id); } while (!shutdown_in_progress(knet_h)) { nev = epoll_wait(knet_h->send_to_links_epollfd, events, KNET_EPOLL_MAX_EVENTS + 1, -1); if (pthread_rwlock_rdlock(&knet_h->global_rwlock) != 0) { log_debug(knet_h, KNET_SUB_TX, "Unable to get read lock"); continue; } for (i = 0; i < nev; i++) { if (events[i].data.fd == knet_h->hostsockfd[0]) { type = KNET_HEADER_TYPE_HOST_INFO; channel = -1; } else { type = KNET_HEADER_TYPE_DATA; for (channel = 0; channel < KNET_DATAFD_MAX; channel++) { if ((knet_h->sockfd[channel].in_use) && (knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created] == events[i].data.fd)) { break; } } if (channel >= KNET_DATAFD_MAX) { log_debug(knet_h, KNET_SUB_TX, "No available channels"); continue; /* channel not found */ } } if (pthread_mutex_lock(&knet_h->tx_mutex) != 0) { log_debug(knet_h, KNET_SUB_TX, "Unable to get mutex lock"); continue; } _handle_send_to_links(knet_h, &msg, events[i].data.fd, channel, type); pthread_mutex_unlock(&knet_h->tx_mutex); } pthread_rwlock_unlock(&knet_h->global_rwlock); } return NULL; } diff --git a/m4/ax_prog_date.m4 b/m4/ax_prog_date.m4 new file mode 100644 index 00000000..c85f0f24 --- /dev/null +++ b/m4/ax_prog_date.m4 @@ -0,0 +1,137 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_prog_date.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_PROG_DATE() +# +# DESCRIPTION +# +# This macro tries to determine the type of the date (1) command and some +# of its non-standard capabilities. +# +# The type is determined as follow: +# +# * If the version string contains "GNU", then: +# - The variable ax_cv_prog_date_gnu is set to "yes". +# - The variable ax_cv_prog_date_type is set to "gnu". +# +# * If date supports the "-v 1d" option, then: +# - The variable ax_cv_prog_date_bsd is set to "yes". +# - The variable ax_cv_prog_date_type is set to "bsd". +# +# * If both previous checks fail, then: +# - The variable ax_cv_prog_date_type is set to "unknown". +# +# The following capabilities of GNU date are checked: +# +# * If date supports the --date arg option, then: +# - The variable ax_cv_prog_date_gnu_date is set to "yes". +# +# * If date supports the --utc arg option, then: +# - The variable ax_cv_prog_date_gnu_utc is set to "yes". +# +# The following capabilities of BSD date are checked: +# +# * If date supports the -v 1d option, then: +# - The variable ax_cv_prog_date_bsd_adjust is set to "yes". +# +# * If date supports the -r arg option, then: +# - The variable ax_cv_prog_date_bsd_date is set to "yes". +# +# All the aforementioned variables are set to "no" before a check is +# performed. +# +# LICENSE +# +# Copyright (c) 2017 Enrico M. Crisostomo +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 3 + +AC_DEFUN([AX_PROG_DATE], [dnl + AC_CACHE_CHECK([for GNU date], [ax_cv_prog_date_gnu], [ + ax_cv_prog_date_gnu=no + if date --version 2>/dev/null | head -1 | grep -q GNU + then + ax_cv_prog_date_gnu=yes + fi + ]) + AC_CACHE_CHECK([for BSD date], [ax_cv_prog_date_bsd], [ + ax_cv_prog_date_bsd=no + if date -v 1d > /dev/null 2>&1 + then + ax_cv_prog_date_bsd=yes + fi + ]) + AC_CACHE_CHECK([for date type], [ax_cv_prog_date_type], [ + ax_cv_prog_date_type=unknown + if test "x${ax_cv_prog_date_gnu}" = "xyes" + then + ax_cv_prog_date_type=gnu + elif test "x${ax_cv_prog_date_bsd}" = "xyes" + then + ax_cv_prog_date_type=bsd + fi + ]) + AS_VAR_IF([ax_cv_prog_date_gnu], [yes], [ + AC_CACHE_CHECK([whether GNU date supports --date], [ax_cv_prog_date_gnu_date], [ + ax_cv_prog_date_gnu_date=no + if date --date=@1512031231 > /dev/null 2>&1 + then + ax_cv_prog_date_gnu_date=yes + fi + ]) + AC_CACHE_CHECK([whether GNU date supports --utc], [ax_cv_prog_date_gnu_utc], [ + ax_cv_prog_date_gnu_utc=no + if date --utc > /dev/null 2>&1 + then + ax_cv_prog_date_gnu_utc=yes + fi + ]) + ]) + AS_VAR_IF([ax_cv_prog_date_bsd], [yes], [ + AC_CACHE_CHECK([whether BSD date supports -r], [ax_cv_prog_date_bsd_date], [ + ax_cv_prog_date_bsd_date=no + if date -r 1512031231 > /dev/null 2>&1 + then + ax_cv_prog_date_bsd_date=yes + fi + ]) + ]) + AS_VAR_IF([ax_cv_prog_date_bsd], [yes], [ + AC_CACHE_CHECK([whether BSD date supports -v], [ax_cv_prog_date_bsd_adjust], [ + ax_cv_prog_date_bsd_adjust=no + if date -v 1d > /dev/null 2>&1 + then + ax_cv_prog_date_bsd_adjust=yes + fi + ]) + ]) +])dnl AX_PROG_DATE diff --git a/man/Makefile.am b/man/Makefile.am index ec53439e..9f2741b1 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -1,110 +1,111 @@ # # Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. # # Authors: Fabio M. Di Nitto # Federico Simoncelli # # This software licensed under GPL-2.0+, LGPL-2.0+ # MAINTAINERCLEANFILES = Makefile.in include $(top_srcdir)/build-aux/check.mk EXTRA_DIST = kronosnetd.8 knet-keygen.8 # Avoid Automake warnings about overriding these user variables. # Programs in this directory are used during the build only. AUTOMAKE_OPTIONS = -Wno-gnu EXEEXT=$(BUILD_EXEEXT) CC=$(CC_FOR_BUILD) CFLAGS=$(CFLAGS_FOR_BUILD) CPPFLAGS=$(CPPFLAGS_FOR_BUILD) LDFLAGS=$(LDFLAGS_FOR_BUILD) if BUILD_MAN if BUILD_KRONOSNETD man8_MANS = kronosnetd.8 knet-keygen.8 endif noinst_PROGRAMS = doxyxml doxyxml_SOURCES = doxyxml.c doxyxml_CFLAGS = $(AM_CFLAGS) $(libqb_BUILD_CFLAGS) $(libxml_BUILD_CFLAGS) doxyxml_LDADD = $(libqb_BUILD_LIBS) $(libxml_BUILD_LIBS) knet_man3_MANS = \ knet_addrtostr.3 \ knet_handle_add_datafd.3 \ knet_handle_clear_stats.3 \ knet_handle_compress.3 \ knet_handle_crypto.3 \ knet_handle_enable_filter.3 \ knet_handle_enable_pmtud_notify.3 \ knet_handle_enable_sock_notify.3 \ knet_handle_free.3 \ knet_handle_get_channel.3 \ knet_get_compress_list.3 \ knet_get_crypto_list.3 \ knet_handle_get_datafd.3 \ knet_handle_get_stats.3 \ knet_get_transport_id_by_name.3 \ knet_get_transport_list.3 \ knet_get_transport_name_by_id.3 \ knet_handle_get_transport_reconnect_interval.3 \ knet_handle_new.3 \ knet_handle_new_ex.3 \ knet_handle_pmtud_get.3 \ knet_handle_pmtud_getfreq.3 \ knet_handle_pmtud_setfreq.3 \ knet_handle_remove_datafd.3 \ knet_handle_setfwd.3 \ knet_handle_set_transport_reconnect_interval.3 \ knet_host_add.3 \ knet_host_enable_status_change_notify.3 \ knet_host_get_host_list.3 \ knet_host_get_id_by_host_name.3 \ knet_host_get_name_by_host_id.3 \ knet_host_get_policy.3 \ knet_host_get_status.3 \ knet_host_remove.3 \ knet_host_set_name.3 \ knet_host_set_policy.3 \ knet_link_clear_config.3 \ knet_link_get_config.3 \ knet_link_get_enable.3 \ knet_link_get_link_list.3 \ knet_link_get_ping_timers.3 \ knet_link_get_pong_count.3 \ knet_link_get_priority.3 \ knet_link_get_status.3 \ knet_link_set_config.3 \ knet_link_set_enable.3 \ knet_link_set_ping_timers.3 \ knet_link_set_pong_count.3 \ knet_link_set_priority.3 \ knet_log_get_loglevel.3 \ knet_log_get_loglevel_id.3 \ knet_log_get_loglevel_name.3 \ knet_log_get_subsystem_id.3 \ knet_log_get_subsystem_name.3 \ knet_log_set_loglevel.3 \ knet_recv.3 \ knet_send.3 \ knet_send_sync.3 \ knet_strtoaddr.3 man3_MANS = $(knet_man3_MANS) $(MANS): doxyfile-knet.stamp doxyfile-knet.stamp: $(builddir)/doxyxml Doxyfile-knet $(top_srcdir)/libknet/libknet.h $(DOXYGEN) Doxyfile-knet - $(builddir)/doxyxml -m -P -o $(builddir) -s 3 -p @PACKAGE_NAME@ -H "Kronosnet Programmer's Manual" -d $(builddir)/xml-knet/ libknet_8h.xml + $(builddir)/doxyxml -m -P -o $(builddir) -s 3 -p @PACKAGE_NAME@ -H "Kronosnet Programmer's Manual" \ + $$($(UTC_DATE_AT)$(SOURCE_EPOCH) +"-D %F -Y %Y") -d $(builddir)/xml-knet/ libknet_8h.xml touch doxyfile-knet.stamp endif clean-local: rm -rf doxyfile*.stamp xml* *.3 diff --git a/man/doxyxml.c b/man/doxyxml.c index ebdda868..fdd2d072 100644 --- a/man/doxyxml.c +++ b/man/doxyxml.c @@ -1,829 +1,856 @@ /* * Copyright (C) 2018 Red Hat, Inc. All rights reserved. * * Author: Christine Caulfield * * This software licensed under GPL-2.0+, LGPL-2.0+ */ /* * NOTE: this code is very rough, it does the bare minimum to parse the * XML out from doxygen and is probably very fragile to changes in that XML * schema. It probably leaks memory all over the place too. * * In its favour, it *does* generate man pages and should only be run very ocasionally */ #define _DEFAULT_SOURCE #define _BSD_SOURCE #define _XOPEN_SOURCE #define _XOPEN_SOURCE_EXTENDED #include #include #include #include #include #include #include #include #include #include #include #define XML_DIR "../../libknet/man/xml" #define XML_FILE "libknet_8h.xml" static int print_ascii = 1; static int print_man = 0; static int print_params = 0; static int num_functions = 0; static const char *man_section="3"; static const char *package_name="Kronosnet"; static const char *header="Kronosnet Programmer's Manual"; static const char *output_dir="./"; static const char *xml_dir = XML_DIR; static const char *xml_file = XML_FILE; +static const char *manpage_date = NULL; +static long manpage_year = LONG_MIN; static struct qb_list_head params_list; static struct qb_list_head retval_list; static qb_map_t *function_map; static qb_map_t *structures_map; static qb_map_t *used_structures_map; struct param_info { char *paramname; char *paramtype; char *paramdesc; struct param_info *next; struct qb_list_head list; }; struct struct_info { char *structname; struct qb_list_head params_list; /* our params */ struct qb_list_head list; }; static char *get_texttree(int *type, xmlNode *cur_node, char **returntext); static void traverse_node(xmlNode *parentnode, const char *leafname, void (do_members(xmlNode*, void*)), void *arg); static void free_paraminfo(struct param_info *pi) { free(pi->paramname); free(pi->paramtype); free(pi->paramdesc); free(pi); } static char *get_attr(xmlNode *node, const char *tag) { xmlAttr *this_attr; for (this_attr = node->properties; this_attr; this_attr = this_attr->next) { if (this_attr->type == XML_ATTRIBUTE_NODE && strcmp((char *)this_attr->name, tag) == 0) { return strdup((char *)this_attr->children->content); } } return NULL; } static char *get_child(xmlNode *node, const char *tag) { xmlNode *this_node; xmlNode *child; char buffer[1024] = {'\0'}; char *refid = NULL; char *declname = NULL; for (this_node = node->children; this_node; this_node = this_node->next) { if ((strcmp( (char*)this_node->name, "declname") == 0)) { declname = strdup((char*)this_node->children->content); } if ((this_node->type == XML_ELEMENT_NODE && this_node->children) && ((strcmp((char *)this_node->name, tag) == 0))) { refid = NULL; for (child = this_node->children; child; child = child->next) { if (child->content) { strcat(buffer, (char *)child->content); } if ((strcmp( (char*)child->name, "ref") == 0)) { if (child->children->content) { strcat(buffer,(char *)child->children->content); } refid = get_attr(child, "refid"); } } } if (declname && refid) { qb_map_put(used_structures_map, refid, declname); } } return strdup(buffer); } static struct param_info *find_param_by_name(struct qb_list_head *list, const char *name) { struct qb_list_head *iter; struct param_info *pi; qb_list_for_each(iter, list) { pi = qb_list_entry(iter, struct param_info, list); if (strcmp(pi->paramname, name) == 0) { return pi; } } return NULL; } static int not_all_whitespace(char *string) { unsigned int i; for (i=0; ichildren; this_tag; this_tag = this_tag->next) { for (sub_tag = this_tag->children; sub_tag; sub_tag = sub_tag->next) { if (sub_tag->type == XML_ELEMENT_NODE && strcmp((char *)sub_tag->name, "parameternamelist") == 0) { paramname = (char*)sub_tag->children->next->children->content; } if (sub_tag->type == XML_ELEMENT_NODE && strcmp((char *)sub_tag->name, "parameterdescription") == 0) { paramdesc = (char*)sub_tag->children->next->children->content; /* Add text to the param_map */ pi = find_param_by_name(list, paramname); if (pi) { pi->paramdesc = paramdesc; } else { pi = malloc(sizeof(struct param_info)); if (pi) { pi->paramname = paramname; pi->paramdesc = paramdesc; pi->paramtype = NULL; /* retval */ qb_list_add_tail(&pi->list, list); } } } } } } static char *get_text(xmlNode *cur_node, char **returntext) { xmlNode *this_tag; xmlNode *sub_tag; char *kind; char buffer[4096] = {'\0'}; for (this_tag = cur_node->children; this_tag; this_tag = this_tag->next) { if (this_tag->type == XML_TEXT_NODE && strcmp((char *)this_tag->name, "text") == 0) { if (not_all_whitespace((char*)this_tag->content)) { strcat(buffer, (char*)this_tag->content); strcat(buffer, "\n"); } } if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "emphasis") == 0) { if (print_man) { strcat(buffer, "\\fB"); } strcat(buffer, (char*)this_tag->children->content); if (print_man) { strcat(buffer, "\\fR"); } } if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "itemizedlist") == 0) { for (sub_tag = this_tag->children; sub_tag; sub_tag = sub_tag->next) { if (sub_tag->type == XML_ELEMENT_NODE && strcmp((char *)sub_tag->name, "listitem") == 0) { strcat(buffer, (char*)sub_tag->children->children->content); strcat(buffer, "\n"); } } } /* Look for subsections - return value & params */ if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "simplesect") == 0) { char *tmp; kind = get_attr(this_tag, "kind"); tmp = get_text(this_tag->children, NULL); if (returntext && strcmp(kind, "return") == 0) { *returntext = tmp; } } if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "parameterlist") == 0) { kind = get_attr(this_tag, "kind"); if (strcmp(kind, "param") == 0) { get_param_info(this_tag, ¶ms_list); } if (strcmp(kind, "retval") == 0) { get_param_info(this_tag, &retval_list); } } } return strdup(buffer); } /* Called from traverse_node() */ static void read_struct(xmlNode *cur_node, void *arg) { xmlNode *this_tag; struct struct_info *si=arg; struct param_info *pi; char fullname[1024]; char *type = NULL; char *name = NULL; const char *args=""; for (this_tag = cur_node->children; this_tag; this_tag = this_tag->next) { if (strcmp((char*)this_tag->name, "type") == 0) { type = (char*)this_tag->children->content ; } if (strcmp((char*)this_tag->name, "name") == 0) { name = (char*)this_tag->children->content ; } if (this_tag->children && strcmp((char*)this_tag->name, "argsstring") == 0) { args = (char*)this_tag->children->content; } } if (name) { pi = malloc(sizeof(struct param_info)); if (pi) { snprintf(fullname, sizeof(fullname), "%s%s", name, args); pi->paramtype = strdup(type); pi->paramname = strdup(fullname); pi->paramdesc = NULL; qb_list_add_tail(&pi->list, &si->params_list); } } } static int read_structure_from_xml(char *refid, char *name) { char fname[PATH_MAX]; xmlNode *rootdoc; xmlDocPtr doc; struct struct_info *si; int ret = -1; snprintf(fname, sizeof(fname), "%s/%s.xml", xml_dir, refid); doc = xmlParseFile(fname); if (doc == NULL) { fprintf(stderr, "Error: unable to open xml file for %s\n", refid); return -1; } rootdoc = xmlDocGetRootElement(doc); if (!rootdoc) { fprintf(stderr, "Can't find \"document root\"\n"); return -1; } si = malloc(sizeof(struct struct_info)); if (si) { qb_list_init(&si->params_list); si->structname = strdup(name); traverse_node(rootdoc, "memberdef", read_struct, si); ret = 0; qb_map_put(structures_map, refid, si); } xmlFreeDoc(doc); return ret; } /* Reformat pointer params so they look nicer */ static void print_param(FILE *manfile, struct param_info *pi, int field_width, int bold, const char *delimiter) { char asterisk = ' '; char *type = pi->paramtype; if (pi->paramtype[strlen(pi->paramtype)-1] == '*') { asterisk='*'; type = strdup(pi->paramtype); type[strlen(type)-1] = '\0'; } fprintf(manfile, " %s%-*s%c%s\\fI%s\\fP%s\n", bold?"\\fB":"", field_width, type, asterisk, bold?"\\fP":"", pi->paramname, delimiter); if (type != pi->paramtype) { free(type); } } static void print_structure(FILE *manfile, char *refid, char *name) { struct struct_info *si; struct param_info *pi; struct qb_list_head *iter; unsigned int max_param_length=0; /* If it's not been read in - go and look for it */ si = qb_map_get(structures_map, refid); if (!si) { if (!read_structure_from_xml(refid, name)) { si = qb_map_get(structures_map, refid); } } if (si) { qb_list_for_each(iter, &si->params_list) { pi = qb_list_entry(iter, struct param_info, list); if (strlen(pi->paramtype) > max_param_length) { max_param_length = strlen(pi->paramtype); } } fprintf(manfile, "struct %s {\n", si->structname); qb_list_for_each(iter, &si->params_list) { pi = qb_list_entry(iter, struct param_info, list); print_param(manfile, pi, max_param_length, 0,";"); } fprintf(manfile, "};\n"); } } char *get_texttree(int *type, xmlNode *cur_node, char **returntext) { xmlNode *this_tag; char *tmp = NULL; char buffer[4096] = {'\0'}; for (this_tag = cur_node->children; this_tag; this_tag = this_tag->next) { if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "para") == 0) { tmp = get_text(this_tag, returntext); strcat(buffer, tmp); strcat(buffer, "\n"); free(tmp); } } if (buffer[0]) { tmp = strdup(buffer); } return tmp; } /* The text output is VERY basic and just a check that it's working really */ static void print_text(char *name, char *def, char *brief, char *args, char *detailed, struct qb_list_head *param_list, char *returntext) { printf(" ------------------ %s --------------------\n", name); printf("NAME\n"); printf(" %s - %s\n", name, brief); printf("SYNOPSIS\n"); printf(" %s %s\n\n", name, args); printf("DESCRIPTION\n"); printf(" %s\n", detailed); if (returntext) { printf("RETURN VALUE\n"); printf(" %s\n", returntext); } } /* Print a long string with para marks in it. */ static void man_print_long_string(FILE *manfile, char *text) { char *next_nl; char *current = text; next_nl = strchr(text, '\n'); while (next_nl && *next_nl != '\0') { *next_nl = '\0'; if (strlen(current)) { fprintf(manfile, ".PP\n%s\n", current); } *next_nl = '\n'; current = next_nl+1; next_nl = strchr(current, '\n'); } } static void print_manpage(char *name, char *def, char *brief, char *args, char *detailed, struct qb_list_head *param_map, char *returntext) { char manfilename[PATH_MAX]; char gendate[64]; + const char *dateptr = gendate; FILE *manfile; time_t t; struct tm *tm; qb_map_iter_t *map_iter; struct qb_list_head *iter; struct qb_list_head *tmp; const char *p; void *data; unsigned int max_param_type_len; unsigned int max_param_name_len; unsigned int num_param_descs; int param_count = 0; int param_num = 0; struct param_info *pi; t = time(NULL); tm = localtime(&t); if (!tm) { perror("unable to get localtime"); exit(1); } strftime(gendate, sizeof(gendate), "%Y-%m-%d", tm); + if (manpage_date) { + dateptr = manpage_date; + } + if (manpage_year == LONG_MIN) { + manpage_year = tm->tm_year+1900; + } + snprintf(manfilename, sizeof(manfilename), "%s/%s.%s", output_dir, name, man_section); manfile = fopen(manfilename, "w+"); if (!manfile) { perror("unable to open output file"); printf("%s", manfilename); exit(1); } /* Work out the length of the parameters, so we can line them up */ max_param_type_len = 0; max_param_name_len = 0; num_param_descs = 0; qb_list_for_each(iter, ¶ms_list) { pi = qb_list_entry(iter, struct param_info, list); if (strlen(pi->paramtype) > max_param_type_len) { max_param_type_len = strlen(pi->paramtype); } if (strlen(pi->paramname) > max_param_name_len) { max_param_name_len = strlen(pi->paramname); } if (pi->paramdesc) { num_param_descs++; } param_count++; } /* Off we go */ fprintf(manfile, ".\\\" Automatically generated man page, do not edit\n"); - fprintf(manfile, ".TH %s %s %s \"%s\" \"%s\"\n", name, man_section, gendate, package_name, header); + fprintf(manfile, ".TH %s %s %s \"%s\" \"%s\"\n", name, man_section, dateptr, package_name, header); fprintf(manfile, ".SH NAME\n"); fprintf(manfile, "%s \\- %s\n", name, brief); fprintf(manfile, ".SH SYNOPSIS\n"); fprintf(manfile, ".nf\n"); fprintf(manfile, ".B #include \n"); fprintf(manfile, ".sp\n"); fprintf(manfile, "\\fB%s\\fP(\n", def); qb_list_for_each(iter, ¶ms_list) { pi = qb_list_entry(iter, struct param_info, list); print_param(manfile, pi, max_param_type_len, 1, ++param_num < param_count?",":""); } fprintf(manfile, ");\n"); fprintf(manfile, ".fi\n"); if (print_params && num_param_descs) { fprintf(manfile, ".SH PARAMS\n"); qb_list_for_each(iter, ¶ms_list) { pi = qb_list_entry(iter, struct param_info, list); fprintf(manfile, "\\fB%-*s \\fP\\fI%s\\fP\n", (int)max_param_name_len, pi->paramname, pi->paramdesc); fprintf(manfile, ".PP\n"); } } fprintf(manfile, ".SH DESCRIPTION\n"); man_print_long_string(manfile, detailed); if (qb_map_count_get(used_structures_map)) { fprintf(manfile, ".SH STRUCTURES\n"); map_iter = qb_map_iter_create(used_structures_map); for (p = qb_map_iter_next(map_iter, &data); p; p = qb_map_iter_next(map_iter, &data)) { fprintf(manfile, ".SS \"\"\n"); fprintf(manfile, ".PP\n"); fprintf(manfile, ".sp\n"); fprintf(manfile, ".sp\n"); fprintf(manfile, ".RS\n"); fprintf(manfile, ".nf\n"); fprintf(manfile, "\\fB\n"); print_structure(manfile, (char*)p, (char *)data); fprintf(manfile, "\\fP\n"); fprintf(manfile, ".fi\n"); } qb_map_iter_free(map_iter); fprintf(manfile, ".RE\n"); } if (returntext) { fprintf(manfile, ".SH RETURN VALUE\n"); man_print_long_string(manfile, returntext); } qb_list_for_each(iter, &retval_list) { pi = qb_list_entry(iter, struct param_info, list); fprintf(manfile, "\\fB%-*s \\fP\\fI%s\\fP\n", 10, pi->paramname, pi->paramdesc); fprintf(manfile, ".PP\n"); } fprintf(manfile, ".SH SEE ALSO\n"); fprintf(manfile, ".PP\n"); fprintf(manfile, ".nh\n"); fprintf(manfile, ".ad l\n"); param_num = 0; map_iter = qb_map_iter_create(function_map); for (p = qb_map_iter_next(map_iter, &data); p; p = qb_map_iter_next(map_iter, &data)) { /* Exclude us! */ if (strcmp(data, name)) { fprintf(manfile, "\\fI%s(%s)%s", (char *)data, man_section, param_num < (num_functions - 1)?", ":""); } param_num++; } qb_map_iter_free(map_iter); fprintf(manfile, "\n"); fprintf(manfile, ".ad\n"); fprintf(manfile, ".hy\n"); fprintf(manfile, ".SH \"COPYRIGHT\"\n"); fprintf(manfile, ".PP\n"); - fprintf(manfile, "Copyright (C) 2010-%4d Red Hat, Inc. All rights reserved.\n", tm->tm_year+1900); + fprintf(manfile, "Copyright (C) 2010-%4ld Red Hat, Inc. All rights reserved.\n", manpage_year); fclose(manfile); /* Free the params & retval info */ qb_list_for_each_safe(iter, tmp, ¶ms_list) { pi = qb_list_entry(iter, struct param_info, list); qb_list_del(&pi->list); free_paraminfo(pi); } qb_list_for_each_safe(iter, tmp, &retval_list) { pi = qb_list_entry(iter, struct param_info, list); qb_list_del(&pi->list); free_paraminfo(pi); } /* Free used-structures map */ map_iter = qb_map_iter_create(used_structures_map); for (p = qb_map_iter_next(map_iter, &data); p; p = qb_map_iter_next(map_iter, &data)) { qb_map_rm(used_structures_map, p); free(data); } } /* Same as traverse_members, but to collect function names */ static void collect_functions(xmlNode *cur_node, void *arg) { xmlNode *this_tag; char *kind; char *name = NULL; if (cur_node->name && strcmp((char *)cur_node->name, "memberdef") == 0) { kind = get_attr(cur_node, "kind"); if (kind && strcmp(kind, "function") == 0) { for (this_tag = cur_node->children; this_tag; this_tag = this_tag->next) { if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "name") == 0) { name = strdup((char *)this_tag->children->content); } } if (name) { qb_map_put(function_map, name, name); num_functions++; } } } } static void traverse_members(xmlNode *cur_node, void *arg) { xmlNode *this_tag; if (cur_node->name && strcmp((char *)cur_node->name, "memberdef") == 0) { char *kind = NULL; char *def = NULL; char *args = NULL; char *name = NULL; char *brief = NULL; char *detailed = NULL; char *returntext = NULL; int type; kind=def=args=name=NULL; kind = get_attr(cur_node, "kind"); for (this_tag = cur_node->children; this_tag; this_tag = this_tag->next) { if (!this_tag->children || !this_tag->children->content) continue; if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "definition") == 0) def = strdup((char *)this_tag->children->content); if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "argsstring") == 0) args = strdup((char *)this_tag->children->content); if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "name") == 0) name = strdup((char *)this_tag->children->content); if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "briefdescription") == 0) { brief = get_texttree(&type, this_tag, &returntext); if (brief) { /* * apparently brief text contains extra trailing space and 2 \n. * remove them. */ brief[strlen(brief) - 3] = '\0'; } } if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "detaileddescription") == 0) { detailed = get_texttree(&type, this_tag, &returntext); } /* Get all the params */ if (this_tag->type == XML_ELEMENT_NODE && strcmp((char *)this_tag->name, "param") == 0) { char *param_type = get_child(this_tag, "type"); char *param_name = get_child(this_tag, "declname"); struct param_info *pi = malloc(sizeof(struct param_info)); if (pi) { pi->paramname = param_name; pi->paramtype = param_type; pi->paramdesc = NULL; qb_list_add_tail(&pi->list, ¶ms_list); } } } if (kind && strcmp(kind, "typedef") == 0) { /* Collect typedefs? */ } if (kind && strcmp(kind, "function") == 0) { if (print_man) { print_manpage(name, def, brief, args, detailed, ¶ms_list, returntext); } else { print_text(name, def, brief, args, detailed, ¶ms_list, returntext); } } free(kind); free(def); free(args); // free(name); /* don't free, it's in the map */ } } static void traverse_node(xmlNode *parentnode, const char *leafname, void (do_members(xmlNode*, void*)), void *arg) { xmlNode *cur_node; for (cur_node = parentnode->children; cur_node; cur_node = cur_node->next) { if (cur_node->type == XML_ELEMENT_NODE && cur_node->name && strcmp((char*)cur_node->name, leafname)==0) { do_members(cur_node, arg); continue; } if (cur_node->type == XML_ELEMENT_NODE) { traverse_node(cur_node, leafname, do_members, arg); } } } static void usage(char *name) { printf("Usage:\n"); - printf(" %s -[am] [-s
] [-p] [-H
] [-o ] []\n", name); + printf(" %s [OPTIONS] []\n", name); printf("\n"); printf(" defaults to %s\n", XML_FILE); printf("\n"); printf(" -a Print ASCII dump of man pages to stdout\n"); printf(" -m Write man page files to \n"); printf(" -P Print PARAMS section\n"); printf(" -s Write man pages into section Use name. default \n"); printf(" -H
Set header (default \"Kronosnet Programmer's Manual\"\n"); + printf(" -D Date to print at top of man pages (format not checked, default: today)\n"); + printf(" -Y Year to print at end of copyright line (default: today's year)\n"); printf(" -o Write all man pages to (default .)\n"); printf(" -d Directory for XML files (default %s)\n", XML_DIR); printf(" -h Print this usage text\n"); } int main(int argc, char *argv[]) { xmlNode *rootdoc; xmlDocPtr doc; int quiet=0; int opt; char xml_filename[PATH_MAX]; - while ( (opt = getopt_long(argc, argv, "H:amPs:d:o:p:f:h?", NULL, NULL)) != EOF) + while ( (opt = getopt_long(argc, argv, "H:amPD:Y:s:d:o:p:f:h?", NULL, NULL)) != EOF) { switch(opt) { case 'a': print_ascii = 1; print_man = 0; break; case 'm': print_man = 1; print_ascii = 0; break; case 'P': print_params = 1; break; case 's': man_section = optarg; break; case 'd': xml_dir = optarg; break; + case 'D': + manpage_date = optarg; + break; + case 'Y': + manpage_year = strtol(optarg, NULL, 10); + /* + * Don't make too many assumptions about the year. I was on call at the + * 2000 rollover. #experience + */ + if (manpage_year == LONG_MIN || manpage_year == LONG_MAX || + manpage_year < 1900) { + fprintf(stderr, "Value passed to -Y is not a valid year number\n"); + return 1; + } + break; case 'p': package_name = optarg; break; case 'H': header = optarg; break; case 'o': output_dir = optarg; break; case '?': case 'h': usage(argv[0]); return 0; } } if (argv[optind]) { xml_file = argv[optind]; } if (!quiet) { fprintf(stderr, "reading xml ... "); } snprintf(xml_filename, sizeof(xml_filename), "%s/%s", xml_dir, xml_file); doc = xmlParseFile(xml_filename); if (doc == NULL) { fprintf(stderr, "Error: unable to read xml file %s\n", xml_filename); exit(1); } rootdoc = xmlDocGetRootElement(doc); if (!rootdoc) { fprintf(stderr, "Can't find \"document root\"\n"); exit(1); } if (!quiet) fprintf(stderr, "done.\n"); qb_list_init(¶ms_list); qb_list_init(&retval_list); structures_map = qb_hashtable_create(10); function_map = qb_hashtable_create(10); used_structures_map = qb_hashtable_create(10); /* Collect functions */ traverse_node(rootdoc, "memberdef", collect_functions, NULL); /* print pages */ traverse_node(rootdoc, "memberdef", traverse_members, NULL); return 0; }