diff --git a/Makefile.am b/Makefile.am index 6d7089bc..321a4958 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,216 +1,211 @@ # # Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. # # Author: Fabio M. Di Nitto # # This software licensed under GPL-2.0+ # MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure depcomp \ config.guess config.sub missing install-sh \ ltmain.sh compile config.h.in config.h.in~ \ autoscan.log configure.scan test-driver \ m4/libtool.m4 m4/lt~obsolete.m4 m4/ltoptions.m4 \ m4/ltsugar.m4 m4/ltversion.m4 include $(top_srcdir)/build-aux/check.mk AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I m4 SPEC = $(PACKAGE_NAME).spec TARGZFILE = $(PACKAGE_NAME)-$(VERSION).tar.gz EXTRA_DIST = autogen.sh .version \ NOTES_TO_PACKAGE_MAINTAINERS \ $(SPEC).in build-aux SUBDIRS = libnozzle libknet if BUILD_MAN SUBDIRS += man endif dist_doc_DATA = \ COPYING.applications \ COPYING.libraries \ COPYRIGHT \ README.licence \ README all-local: $(SPEC) clean-local: rm -rf $(SPEC) cov* distclean-local: rm -f $(PACKAGE_NAME)-*.tar.* $(PACKAGE_NAME)-*.sha256* tag-* ## make rpm/srpm section. if BUILD_FOR_SOLARIS $(SPEC): $(SPEC).in .version config.status echo "RPM NOT SUPPORTED. skipping" touch $(SPEC) else $(SPEC): $(SPEC).in .version config.status rm -f $@-t $@ date="`LC_ALL=C $(UTC_DATE_AT)$(SOURCE_EPOCH) "+%a %b %d %Y"`" && \ gvgver="`cd $(abs_srcdir); build-aux/git-version-gen --fallback $(VERSION) .tarball-version .gitarchivever`" && \ if [ "$$gvgver" = "`echo $$gvgver | sed 's/-/./'`" ];then \ rpmver="$$gvgver" && \ alphatag="" && \ dirty="" && \ numcomm="0"; \ else \ gitver="`echo $$gvgver | sed 's/\(.*\)\./\1-/'`" && \ rpmver=`echo $$gitver | sed 's/-.*//g'` && \ alphatag=`echo $$gvgver | sed 's/[^-]*-\([^-]*\).*/\1/'` && \ numcomm=`echo $$gitver | sed 's/[^-]*-\([^-]*\).*/\1/'` && \ dirty="" && \ if [ "`echo $$gitver | sed 's/^.*-dirty$$//g'`" = "" ];then \ dirty="dirty"; \ fi \ fi && \ if [ -n "$$dirty" ]; then dirty="dirty"; else dirty=""; fi && \ if [ "$$numcomm" = "0" ]; then \ sed \ -e "s#@version@#$$rpmver#g" \ -e "s#%glo.*alpha.*##g" \ -e "s#%glo.*numcomm.*##g" \ -e "s#@dirty@#$$dirty#g" \ -e "s#@date@#$$date#g" \ $(abs_srcdir)/$@.in > $@-t; \ else \ sed \ -e "s#@version@#$$rpmver#g" \ -e "s#@alphatag@#$$alphatag#g" \ -e "s#@numcomm@#$$numcomm#g" \ -e "s#@dirty@#$$dirty#g" \ -e "s#@date@#$$date#g" \ $(abs_srcdir)/$@.in > $@-t; \ fi; \ if [ -z "$$dirty" ]; then sed -i -e "s#%glo.*dirty.*##g" $@-t; fi -if BUILD_SCTP - sed -i -e "s#@sctp@#bcond_without#g" $@-t -else - sed -i -e "s#@sctp@#bcond_with#g" $@-t -endif if BUILD_CRYPTO_NSS sed -i -e "s#@nss@#bcond_without#g" $@-t else sed -i -e "s#@nss@#bcond_with#g" $@-t endif if BUILD_CRYPTO_OPENSSL sed -i -e "s#@openssl@#bcond_without#g" $@-t else sed -i -e "s#@openssl@#bcond_with#g" $@-t endif if BUILD_CRYPTO_GCRYPT sed -i -e "s#@gcrypt@#bcond_without#g" $@-t else sed -i -e "s#@gcrypt@#bcond_with#g" $@-t endif if BUILD_COMPRESS_ZLIB sed -i -e "s#@zlib@#bcond_without#g" $@-t else sed -i -e "s#@zlib@#bcond_with#g" $@-t endif if BUILD_COMPRESS_LZ4 sed -i -e "s#@lz4@#bcond_without#g" $@-t else sed -i -e "s#@lz4@#bcond_with#g" $@-t endif if BUILD_COMPRESS_LZO2 sed -i -e "s#@lzo2@#bcond_without#g" $@-t else sed -i -e "s#@lzo2@#bcond_with#g" $@-t endif if BUILD_COMPRESS_LZMA sed -i -e "s#@lzma@#bcond_without#g" $@-t else sed -i -e "s#@lzma@#bcond_with#g" $@-t endif if BUILD_COMPRESS_BZIP2 sed -i -e "s#@bzip2@#bcond_without#g" $@-t else sed -i -e "s#@bzip2@#bcond_with#g" $@-t endif if BUILD_COMPRESS_ZSTD sed -i -e "s#@zstd@#bcond_without#g" $@-t else sed -i -e "s#@zstd@#bcond_with#g" $@-t endif if BUILD_LIBNOZZLE sed -i -e "s#@libnozzle@#bcond_without#g" $@-t else sed -i -e "s#@libnozzle@#bcond_with#g" $@-t endif if BUILD_RUNAUTOGEN sed -i -e "s#@runautogen@#bcond_without#g" $@-t else sed -i -e "s#@runautogen@#bcond_with#g" $@-t endif if OVERRIDE_RPM_DEBUGINFO sed -i -e "s#@overriderpmdebuginfo@#bcond_without#g" $@-t else sed -i -e "s#@overriderpmdebuginfo@#bcond_with#g" $@-t endif if BUILD_RPM_DEBUGINFO sed -i -e "s#@rpmdebuginfo@#bcond_without#g" $@-t else sed -i -e "s#@rpmdebuginfo@#bcond_with#g" $@-t endif if BUILD_MAN sed -i -e "s#@buildman@#bcond_without#g" $@-t else sed -i -e "s#@buildman@#bcond_with#g" $@-t endif if INSTALL_TESTS sed -i -e "s#@installtests@#bcond_without#g" $@-t else sed -i -e "s#@installtests@#bcond_with#g" $@-t endif sed -i -e "s#@defaultadmgroup@#$(DEFAULTADMGROUP)#g" $@-t chmod a-w $@-t mv $@-t $@ rm -f $@-t* endif $(TARGZFILE): $(MAKE) dist RPMBUILDOPTS = --define "_sourcedir $(abs_builddir)" \ --define "_specdir $(abs_builddir)" \ --define "_builddir $(abs_builddir)" \ --define "_srcrpmdir $(abs_builddir)" \ --define "_rpmdir $(abs_builddir)" srpm: clean $(MAKE) $(SPEC) $(TARGZFILE) rpmbuild $(RPMBUILDOPTS) --nodeps -bs $(SPEC) rpm: clean $(MAKE) $(SPEC) $(TARGZFILE) rpmbuild $(RPMBUILDOPTS) -ba $(SPEC) # release/versioning BUILT_SOURCES = .version .version: echo $(VERSION) > $@-t && mv $@-t $@ dist-hook: gen-ChangeLog echo $(VERSION) > $(distdir)/.tarball-version echo $(SOURCE_EPOCH) > $(distdir)/source_epoch gen_start_date = 2000-01-01 .PHONY: gen-ChangeLog gen-ChangeLog: if test -d $(abs_srcdir)/.git; then \ LC_ALL=C $(top_srcdir)/build-aux/gitlog-to-changelog \ --since=$(gen_start_date) > $(distdir)/cl-t; \ rm -f $(distdir)/ChangeLog; \ mv $(distdir)/cl-t $(distdir)/ChangeLog; \ fi diff --git a/README b/README index dce36f55..d080771b 100644 --- a/README +++ b/README @@ -1,106 +1,100 @@ # # Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. # # Author: Fabio M. Di Nitto # # This software licensed under GPL-2.0+ # Upstream resources ------------------ https://github.com/kronosnet/kronosnet/ https://ci.kronosnet.org/ https://projects.clusterlabs.org/project/board/86/ (TODO list and activities tracking) https://drive.google.com/drive/folders/0B_zxAPgZTkM_TklfYzN6a2FYUFE?resourcekey=0-Cfr5D94rZ8LVbeMPGjxbdg&usp=sharing (google shared drive) https://lists.kronosnet.org/mailman3/postorius/lists/users.lists.kronosnet.org/ https://lists.kronosnet.org/mailman3/postorius/lists/devel.lists.kronosnet.org/ https://lists.kronosnet.org/mailman3/postorius/lists/commits.lists.kronosnet.org/ https://kronosnet.org/ (web 0.1 style) IRC: #kronosnet on Libera.Chat Architecture ------------ Please refer to the google shared drive Presentations directory for diagrams and fancy schemas Dependancies ------------ kronosnet has few obligatory dependancies, though it is strongly recommended that you build it with at least one crypto library enabled. It does, however, require libqb for both the doxygen2man tool (for creating the API man pages) and headers for list manipulation. You can get these from installing libqb from source or libqb-devel package, and your distro might provide doxgen2man as its own package. Running on FreeBSD ----------------------- knet requires big socket buffers and you need to set: kern.ipc.maxsockbuf=18388608 in /etc/sysctl.conf or knet will fail to run. -For version 12 (or lower), knet requires also: -net.inet.sctp.blackhole=1 -in /etc/sysctl.conf or knet will fail to work with SCTP. -This sysctl is obsoleted in version 13. - libnozzle requires if_tap.ko loaded in the kernel. Please avoid using ifconfig_DEFAULT in /etc/rc.conf to use DHCP for all interfaces or the dhclient will interfere with libnozzle interface management, causing errors on some operations such as "ifconfig tap down". Building on Solaris / Illumos ----------------------------- tested on SunOS openindiana-x86-64 5.11 illumos-5b6ecd7fe9: # pkg install autoconf automake libtool pkg-config \ gcc-14 gnu-binutils gnu-coreutils gnu-make \ check system/mozilla-nss doxygen \ header-tun tun optional: # pkg install developer/clang-20 GNU tools must be preferred: # export PATH=/usr/gnu/bin:$PATH mozilla-nss is currently broken in oi-userland and does not ship pkg-config files # ./autogen.sh && ./configure --disable-crypto-nss # make all -j && make check Running on Solaris / Illumos ---------------------------- Tune socket buffers for the protocol you intend to use: -ipadm set-prop -p max_buf=8388608 sctp ipadm set-prop -p max_buf=8388608 udp Rust Bindings ------------- Rust bindings for libknet and libnozzle are part of this source tree, but are included here mainly to keep all of the kronosnet APIs in one place and to ensure that everything is kept up-to-date and properly tested in our CI system. The correct place to get the Rust crates for libknet and libnozzle is still crates.io as it would be for other crates. These will be updated when we issue a new release of knet. https://crates.io/crates/knet-bindings https://crates.io/crates/nozzle-bindings Of course, if you want to try any new features in the APIs that may have not yet been released then you can try these sources, but please keep in touch with us via email or IRC if you do so. diff --git a/configure.ac b/configure.ac index 38e8849c..94a9b15b 100644 --- a/configure.ac +++ b/configure.ac @@ -1,699 +1,674 @@ # # Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. # # Authors: Fabio M. Di Nitto # Federico Simoncelli # # This software licensed under GPL-2.0+ # # -*- Autoconf -*- # Process this file with autoconf to produce a configure script. # AC_PREREQ([2.63]) AC_INIT([kronosnet], m4_esyscmd([build-aux/git-version-gen .tarball-version .gitarchivever]), [devel@lists.kronosnet.org]) # Don't let AC_PROC_CC (invoked by AC_USE_SYSTEM_EXTENSIONS) replace # undefined CFLAGS with -g -O2, overriding our special OPT_CFLAGS. : ${CFLAGS=""} AC_USE_SYSTEM_EXTENSIONS AM_INIT_AUTOMAKE([1.13 dist-bzip2 dist-xz color-tests -Wno-portability subdir-objects]) LT_PREREQ([2.2.6]) AC_CANONICAL_HOST # --enable-new-dtags: Use RUNPATH instead of RPATH. # It is necessary to have this done before libtool does linker detection. # See also: https://github.com/kronosnet/kronosnet/issues/107 # --as-needed: Modern systems have builtin ceil() making -lm superfluous but # AC_SEARCH_LIBS can't detect this because it tests with a false prototype case "$host_os" in solaris*) AC_MSG_NOTICE([Skipping --enable-new-dtags check, Solaris' RPATH logic is like RUNPATH on other platforms]) ;; *) AX_CHECK_LINK_FLAG([-Wl,--enable-new-dtags], [AM_LDFLAGS=-Wl,--enable-new-dtags], [AC_MSG_ERROR(["Linker support for --enable-new-dtags is required"])]) ;; esac AX_CHECK_LINK_FLAG([-Wl,--as-needed], [AM_LDFLAGS="$AM_LDFLAGS -Wl,--as-needed"]) saved_LDFLAGS="$LDFLAGS" LDFLAGS="$AM_LDFLAGS $LDFLAGS" LT_INIT LDFLAGS="$saved_LDFLAGS" AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_SRCDIR([libknet/handle.c]) AC_CONFIG_HEADERS([config.h]) AC_LANG([C]) if test "$prefix" = "NONE"; then prefix="/usr" if test "$localstatedir" = "\${prefix}/var"; then localstatedir="/var" fi if test "$libdir" = "\${exec_prefix}/lib"; then if test -e /usr/lib64; then libdir="/usr/lib64" else libdir="/usr/lib" fi fi fi AC_PROG_AWK AC_PROG_GREP AC_PROG_SED AC_PROG_CPP AC_PROG_CC m4_version_prereq([2.70], [:], [AC_PROG_CC_C99]) if test "x$ac_cv_prog_cc_c99" = "xno"; then AC_MSG_ERROR(["C99 support is required"]) fi AC_PROG_LN_S AC_PROG_INSTALL AC_PROG_MAKE_SET PKG_PROG_PKG_CONFIG AC_CHECK_PROGS([VALGRIND_EXEC], [valgrind]) AM_CONDITIONAL([HAS_VALGRIND], [test x$VALGRIND_EXEC != "x"]) AC_CHECK_PROGS([COVBUILD_EXEC], [cov-build]) AM_CONDITIONAL([HAS_COVBUILD], [test x$COVBUILD_EXEC != "x"]) AC_CHECK_PROGS([COVANALYZE_EXEC], [cov-analyze]) AM_CONDITIONAL([HAS_COVANALYZE], [test x$COVANALYZE_EXEC != "x"]) AC_CHECK_PROGS([COVFORMATERRORS_EXEC], [cov-format-errors]) AM_CONDITIONAL([HAS_COVFORMATERRORS], [test x$COVFORMATERRORS_EXEC != "x"]) # KNET_OPTION_DEFINES(stem,type,detection code) # stem: enters name of option, Automake conditional and preprocessor define # type: compress or crypto, determines where the default comes from AC_DEFUN([KNET_OPTION_DEFINES],[ AC_ARG_ENABLE([$2-$1],[AS_HELP_STRING([--disable-$2-$1],[disable libknet $1 support])],, [enable_$2_$1="$enable_$2_all"]) AM_CONDITIONAL([BUILD_]m4_toupper([$2_$1]),[test "x$enable_$2_$1" = xyes]) if test "x$enable_$2_$1" = xyes; then $3 fi AC_DEFINE_UNQUOTED([WITH_]m4_toupper([$2_$1]), [`test "x$enable_$2_$1" != xyes; echo $?`], $1 $2 [built in]) ]) AC_ARG_ENABLE([man], [AS_HELP_STRING([--disable-man],[disable man page creation])],, [ enable_man="yes" ]) AM_CONDITIONAL([BUILD_MAN], [test x$enable_man = xyes]) -AC_ARG_ENABLE([libknet-sctp], - [AS_HELP_STRING([--disable-libknet-sctp],[disable libknet SCTP support])],, - [ enable_libknet_sctp="yes" ]) -AM_CONDITIONAL([BUILD_SCTP], [test x$enable_libknet_sctp = xyes]) - AC_ARG_ENABLE([functional-tests], [AS_HELP_STRING([--disable-functional-tests],[disable execution of functional tests, useful for old and slow arches])],, [ enable_functional_tests="yes" ]) AM_CONDITIONAL([RUN_FUN_TESTS], [test x$enable_functional_tests = xyes]) AC_ARG_ENABLE([crypto-all], [AS_HELP_STRING([--disable-crypto-all],[disable libknet all crypto modules support])],, [ enable_crypto_all="yes" ]) KNET_OPTION_DEFINES([nss],[crypto],[PKG_CHECK_MODULES([nss], [nss])]) KNET_OPTION_DEFINES([openssl],[crypto],[PKG_CHECK_MODULES([openssl], [libcrypto])]) # use gcry_mac_open to detect if libgcrypt is new enough KNET_OPTION_DEFINES([gcrypt],[crypto],[ PKG_CHECK_MODULES([gcrypt], [libgcrypt >= 1.8.0],, [AC_CHECK_HEADERS([gcrypt.h], [AC_CHECK_LIB([gcrypt], [gcry_mac_open], [AC_SUBST([gcrypt_LIBS], ["-lgcrypt -ldl -lgpg-error"])])], [AC_MSG_ERROR(["missing required gcrypt.h"])])]) ]) AC_ARG_ENABLE([compress-all], [AS_HELP_STRING([--disable-compress-all],[disable libknet all compress modules support])],, [ enable_compress_all="yes" ]) KNET_OPTION_DEFINES([zstd],[compress],[PKG_CHECK_MODULES([libzstd], [libzstd])]) KNET_OPTION_DEFINES([zlib],[compress],[PKG_CHECK_MODULES([zlib], [zlib])]) KNET_OPTION_DEFINES([lz4],[compress],[PKG_CHECK_MODULES([liblz4], [liblz4])]) KNET_OPTION_DEFINES([lzo2],[compress],[ PKG_CHECK_MODULES([lzo2], [lzo2], [# work around broken pkg-config file in v2.10 AC_SUBST([lzo2_CFLAGS],[`echo $lzo2_CFLAGS | sed 's,/lzo *, ,'`])], [AC_CHECK_HEADERS([lzo/lzo1x.h], [AC_CHECK_LIB([lzo2], [lzo1x_decompress_safe], [AC_SUBST([lzo2_LIBS], [-llzo2])])], [AC_MSG_ERROR(["missing required lzo/lzo1x.h header"])])]) ]) KNET_OPTION_DEFINES([lzma],[compress],[PKG_CHECK_MODULES([liblzma], [liblzma])]) KNET_OPTION_DEFINES([bzip2],[compress],[ PKG_CHECK_MODULES([bzip2], [bzip2],, [AC_CHECK_HEADERS([bzlib.h], [AC_CHECK_LIB([bz2], [BZ2_bzBuffToBuffCompress], [AC_SUBST([bzip2_LIBS], [-lbz2])])], [AC_MSG_ERROR(["missing required bzlib.h"])])]) ]) AC_ARG_ENABLE([install-tests], [AS_HELP_STRING([--enable-install-tests],[install tests])],, [ enable_install_tests="no" ]) AM_CONDITIONAL([INSTALL_TESTS], [test x$enable_install_tests = xyes]) AC_ARG_ENABLE([runautogen], [AS_HELP_STRING([--enable-runautogen],[run autogen.sh])],, [ enable_runautogen="no" ]) AM_CONDITIONAL([BUILD_RUNAUTOGEN], [test x$enable_runautogen = xyes]) override_rpm_debuginfo_option="yes" AC_ARG_ENABLE([rpm-debuginfo], [AS_HELP_STRING([--enable-rpm-debuginfo],[build debuginfo packages])],, [ enable_rpm_debuginfo="no", override_rpm_debuginfo_option="no" ]) AM_CONDITIONAL([BUILD_RPM_DEBUGINFO], [test x$enable_rpm_debuginfo = xyes]) AM_CONDITIONAL([OVERRIDE_RPM_DEBUGINFO], [test x$override_rpm_debuginfo_option = xyes]) AC_ARG_ENABLE([libnozzle], [AS_HELP_STRING([--enable-libnozzle],[libnozzle support])],, [ enable_libnozzle="yes" ]) AM_CONDITIONAL([BUILD_LIBNOZZLE], [test x$enable_libnozzle = xyes]) AC_ARG_ENABLE([rust-bindings], [AS_HELP_STRING([--enable-rust-bindings],[rust bindings support])],, [ enable_rust_bindings="no" ]) AM_CONDITIONAL([BUILD_RUST_BINDINGS], [test x$enable_rust_bindings = xyes]) ## local helper functions # this function checks if CC support options passed as # args. Global CPPFLAGS are ignored during this test. cc_supports_flag() { saveCPPFLAGS="$CPPFLAGS" CPPFLAGS="-Werror $@" AC_MSG_CHECKING([whether $CC supports "$@"]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include ], [ #ifdef __USE_FORTIFY_LEVEL printf("%d\n", __USE_FORTIFY_LEVEL) #else printf("hello world\n") #endif ])], [RC=0; AC_MSG_RESULT([yes])], [RC=1; AC_MSG_RESULT([no])]) CPPFLAGS="$saveCPPFLAGS" return $RC } # Checks for libraries. AX_PTHREAD(,[AC_MSG_ERROR([POSIX threads support is required])]) saved_LIBS="$LIBS" LIBS= AC_SEARCH_LIBS([ceil], [m], , [AC_MSG_ERROR([ceil not found])]) AC_SUBST([m_LIBS], [$LIBS]) LIBS= AC_SEARCH_LIBS([clock_gettime], [rt], , [AC_MSG_ERROR([clock_gettime not found])]) AC_SUBST([rt_LIBS], [$LIBS]) LIBS= AC_SEARCH_LIBS([dlopen], [dl dld], , [AC_MSG_ERROR([dlopen not found])]) AC_SUBST([dl_LIBS], [$LIBS]) LIBS="$saved_LIBS" # Check RTLD_DI_ORIGIN (not decalred by musl. glibc has it as an enum so cannot use ifdef) AC_CHECK_DECL([RTLD_DI_ORIGIN], [AC_DEFINE([HAVE_RTLD_DI_ORIGIN], 1, [define when RTLD_DI_ORIGIN is declared])], ,[[#include ]]) AC_CHECK_DECLS([MSG_NOSIGNAL], [], [CFLAGS="$CFLAGS -DMSG_NOSIGNAL=0"], [#include ]) AC_CHECK_LIB([socket], [socket], [AC_SUBST([socket_LIBS], [-lsocket])]) # OS detection build_for_solaris=no AC_MSG_CHECKING([for os in ${host_os}]) case "$host_os" in *linux*) AC_DEFINE_UNQUOTED([KNET_LINUX], [1], [Compiling for Linux platform]) AC_MSG_RESULT([Linux]) ;; *bsd*) AC_DEFINE_UNQUOTED([KNET_BSD], [1], [Compiling for BSD platform]) AC_MSG_RESULT([BSD]) ;; solaris*) AC_DEFINE_UNQUOTED([KNET_SOLARIS], [1], [Compiling for Solaris platform]) AC_MSG_RESULT([Solaris]) build_for_solaris=yes ;; *) AC_MSG_ERROR([Unsupported OS? hmmmm]) ;; esac AM_CONDITIONAL([BUILD_FOR_SOLARIS], [test "x$build_for_solaris" = "xyes"]) # Checks for header files. AC_CHECK_HEADERS([sys/epoll.h]) AC_CHECK_FUNCS([kevent]) # if neither sys/epoll.h nor kevent are present, we should fail. if test "x$ac_cv_header_sys_epoll_h" = xno && test "x$ac_cv_func_kevent" = xno; then AC_MSG_ERROR([Both epoll and kevent unavailable on this OS]) fi if test "x$ac_cv_header_sys_epoll_h" = xyes && test "x$ac_cv_func_kevent" = xyes; then AC_MSG_ERROR([Both epoll and kevent available on this OS, please contact the maintainers to fix the code]) fi -if test "x$enable_libknet_sctp" = xyes; then - AC_CHECK_HEADERS([netinet/sctp.h],, - AC_CACHE_CHECK([for netinet/sctp.h with netinet/in.h included], [ac_cv_have_sctp], [ - ac_cv_have_sctp=no - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ - #include - #include - ], [ - sctp_assoc_t my_sctp_assoc; - ])], - [ac_cv_have_sctp=yes], - [AC_MSG_ERROR(["missing required SCTP headers"])]) - ]) - AC_CHECK_LIB([sctp], [sctp_bindx], [sctp_LIBS="-lsctp"]) - if test "x$ac_cv_have_sctp" = xyes; then - AC_DEFINE([HAVE_NETINET_SCTP_H], [1], [define when HAVE_NETINET_SCTP_H is declared]) - fi - ) -fi - # Checks for typedefs, structures, and compiler characteristics. AC_C_INLINE AC_TYPE_PID_T AC_TYPE_SIZE_T AC_TYPE_SSIZE_T AC_TYPE_UINT8_T AC_TYPE_UINT16_T AC_TYPE_UINT32_T AC_TYPE_UINT64_T AC_TYPE_INT8_T AC_TYPE_INT16_T AC_TYPE_INT32_T AC_TYPE_INT64_T PKG_CHECK_MODULES([libqb], [libqb]) if test "x$enable_man" = "xyes"; then AC_ARG_VAR([DOXYGEN], [override doxygen executable]) AC_CHECK_PROGS([DOXYGEN], [doxygen], [no]) if test "x$DOXYGEN" = xno; then AC_MSG_ERROR(["Doxygen command not found"]) fi AC_ARG_VAR([DOXYGEN2MAN], [override doxygen2man executable]) # required to detect doxygen2man when libqb is installed # in non standard paths saved_PKG_CONFIG="$PKG_CONFIG" saved_ac_cv_path_PKG_CONFIG="$ac_cv_path_PKG_CONFIG" unset PKG_CONFIG ac_cv_path_PKG_CONFIG AC_PATH_PROG([PKG_CONFIG], [pkg-config]) PKG_CHECK_MODULES([libqb_BUILD], [libqb]) PKG_CHECK_VAR([libqb_BUILD_PREFIX], [libqb], [prefix]) AC_PATH_PROG([DOXYGEN2MAN], [doxygen2man], [no], [$libqb_BUILD_PREFIX/bin$PATH_SEPARATOR$PATH]) PKG_CONFIG="$saved_PKG_CONFIG" ac_cv_path_PKG_CONFIG="$saved_ac_cv_path_PKG_CONFIG" if test "x$DOXYGEN2MAN" = "xno"; then AC_MSG_ERROR(["doxygen2man command not found"]) fi AC_SUBST([DOXYGEN2MAN]) fi # check for rust tools to build bindings if test "x$enable_rust_bindings" = "xyes"; then AC_PATH_PROG([CARGO], [cargo], [no]) if test "x$CARGO" = xno; then AC_MSG_ERROR(["cargo command not found"]) fi AC_PATH_PROG([RUSTC], [rustc], [no]) if test "x$RUSTC" = xno; then AC_MSG_ERROR(["rustc command not found"]) fi AC_PATH_PROG([RUSTDOC], [rustdoc], [no]) if test "x$RUSTDOC" = xno; then AC_MSG_ERROR(["rustdoc command not found"]) fi AC_PATH_PROG([BINDGEN], [bindgen], [no]) if test "x$BINDGEN" = xno; then AC_MSG_ERROR(["bindgen command not found"]) fi AC_PATH_PROG([CLIPPY], [clippy-driver], [no]) if test "x$CLIPPY" = xno; then AC_MSG_ERROR(["clippy-driver command not found"]) fi AC_PATH_PROG([RUSTFMT], [rustfmt], [no]) if test "x$RUSTFMT" = xno; then AC_MSG_ERROR(["rustfmt command not found (optional)"]) fi fi # checks for libnozzle if test "x$enable_libnozzle" = xyes; then case $host_os in *linux*) PKG_CHECK_MODULES([libnl], [libnl-3.0]) PKG_CHECK_MODULES([libnlroute], [libnl-route-3.0 >= 3.3], [], [PKG_CHECK_MODULES([libnlroute], [libnl-route-3.0 < 3.3], [AC_DEFINE_UNQUOTED([LIBNL3_WORKAROUND], [1], [Enable libnl < 3.3 build workaround])], [])]) ;; *solaris*) saved_LIBS="$LIBS" AC_SEARCH_LIBS([dlpi_open], [dlpi], , [AC_MSG_ERROR([dlpi_open not found])]) AC_SUBST([dlpi_LIBS], [$LIBS]) LIBS="$saved_LIBS" AC_CHECK_HEADERS([net/if_tun.h],, [AC_MSG_ERROR([net/if_tun.h not found])]) ;; *) ;; esac fi # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html knetcurrent="2" knetrevision="0" knetage="0" # c:r:a libknetversion="$knetcurrent:$knetrevision:$knetage" # soname derived from c:r:a # use $VERSION as build info https://semver.org/. build info are incremental automatically knetalpha="-alpha1" libknetrustver="$(($knetcurrent - $knetage)).$knetage.$knetrevision$knetalpha+$VERSION" nozzlecurrent="1" nozzlerevision="0" nozzleage="0" libnozzleversion="$nozzlecurrent:$nozzlerevision:$nozzleage" # nozzle is stable for now nozzlealpha="" libnozzlerustver="$(($nozzlecurrent - $nozzleage)).$nozzleage.$nozzlerevision$nozzlealpha+$VERSION" AC_SUBST([libknetversion]) AC_SUBST([libknetrustver]) AC_SUBST([libnozzleversion]) AC_SUBST([libnozzlerustver]) # local options AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug],[enable debug build])]) AC_ARG_ENABLE([onwire-v1-extra-debug], [AS_HELP_STRING([--enable-onwire-v1-extra-debug],[enable onwire protocol v1 extra debug. WARNING: IT BREAKS ONWIRE COMPATIBILITY! DO NOT USE IN PRODUCTION!])]) if test "x${enable_onwire_v1_extra_debug}" = xyes; then AC_DEFINE_UNQUOTED([ONWIRE_V1_EXTRA_DEBUG], [1], [Enable crc32 checksum for data and packets]) fi # for standard crc32 function (used in test suite) PKG_CHECK_MODULES([zlib], [zlib]) AC_ARG_ENABLE([hardening], [AS_HELP_STRING([--disable-hardening],[disable hardening build flags])],, [ enable_hardening="yes" ]) AC_ARG_WITH([sanitizers], [AS_HELP_STRING([--with-sanitizers=...,...], [enable SANitizer build, do *NOT* use for production. Only ASAN/UBSAN/TSAN are currently supported])], [ SANITIZERS="$withval" ], [ SANITIZERS="" ]) AC_ARG_WITH([testdir], [AS_HELP_STRING([--with-testdir=DIR],[path to /usr/lib../kronosnet/tests/ dir where to install the test suite])], [ TESTDIR="$withval" ], [ TESTDIR="$libdir/kronosnet/tests" ]) ## do subst AC_SUBST([TESTDIR]) # debug build stuff if test "x${enable_debug}" = xyes; then AC_DEFINE_UNQUOTED([DEBUG], [1], [Compiling Debugging code]) OPT_CFLAGS="-O0" RUST_FLAGS="" RUST_TARGET_DIR="debug" else OPT_CFLAGS="-O3" RUST_FLAGS="--release" RUST_TARGET_DIR="release" fi # check flags for CMSG_SPACE() AC_CACHE_CHECK([for CMSG_SPACE() requiring _XOPEN_SOURCE], [ac_cv_macro_xopen_source], [ ac_cv_macro_xopen_source=yes saveCPPFLAGS="$CPPFLAGS" CPPFLAGS="-Werror $CPPFLAGS" # First, try to link with strdup() using standard flags AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ #include ], [ unsigned char c = CMSG_SPACE(1); ])], [ac_cv_macro_xopen_source=no], [ # Failure, try again with -D_XOPEN_SOURCE XOPEN_CPPFLAGS="-D_XOPEN_SOURCE -D_XOPEN_SOURCE_EXTENDED=1" CPPFLAGS="$CPPFLAGS $XOPEN_CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([ #include ], [ unsigned char c = CMSG_SPACE(1); ], [ac_cv_macro_xopen_source=yes], [AC_MSG_ERROR([no CMSG_SPACE() found])])]) ]) CPPFLAGS="$saveCPPFLAGS" ]) AC_SUBST([XOPEN_CPPFLAGS]) # Check for availablility of hardening options annocheck=no if test "x${enable_hardening}" = xyes; then # support only gcc for now if echo $CC | grep -q gcc; then ANNOPLUGIN="-fplugin=annobin" annocheck=yes fi FORTIFY_CFLAGS="" if test "x${enable_debug}" != xyes; then for j in 3 2; do FORTIFY_CFLAGS_TEMP="-D_FORTIFY_SOURCE=$j" if cc_supports_flag "$OPT_CFLAGS $FORTIFY_CFLAGS_TEMP"; then FORTIFY_CFLAGS="$FORTIFY_CFLAGS_TEMP" break fi done fi HARDENING_CFLAGS_ANNOCHECK="$ANNOPLUGIN -fPIC -DPIC" case "$host_os" in solaris*) # no support for -pie when linking, but supported during compile ;; *) HARDENING_CFLAGS_ANNOCHECK="$HARDENING_CFLAGS_ANNOCHECK -pie" ;; esac HARDENING_CFLAGS_ANNOCHECK="$HARDENING_CFLAGS_ANNOCHECK -fstack-protector-strong -fexceptions -D_GLIBCXX_ASSERTIONS -Wl,-z,now" HARDENING_CFLAGS="-fstack-clash-protection -fcf-protection=full -mcet -mstackrealign" EXTRA_HARDENING_CFLAGS="" # check for annobin required cflags/ldflags for j in $HARDENING_CFLAGS_ANNOCHECK; do if cc_supports_flag $j; then EXTRA_HARDENING_CFLAGS="$EXTRA_HARDENING_CFLAGS $j" else annocheck=no fi done # check for other hardening cflags/ldflags for j in $HARDENING_CFLAGS; do if cc_supports_flag $j; then EXTRA_HARDENING_CFLAGS="$EXTRA_HARDENING_CFLAGS $j" fi done EXTRA_HARDENING_CFLAGS="$EXTRA_HARDENING_CFLAGS $FORTIFY_CFLAGS" # check if annocheck binary is available if test "x${annocheck}" = xyes; then AC_CHECK_PROGS([ANNOCHECK_EXEC], [annocheck]) if test "x${ANNOCHECK_EXEC}" = x; then annocheck=no fi fi AM_LDFLAGS="$AM_LDFLAGS $EXTRA_HARDENING_CFLAGS" fi if test "x${enable_debug}" = xyes; then annocheck=no fi AM_CONDITIONAL([HAS_ANNOCHECK], [test "x$annocheck" = "xyes"]) # gdb flags if test "x${GCC}" = xyes; then GDB_CFLAGS="-ggdb3" else GDB_CFLAGS="-g" fi # --- ASAN/UBSAN/TSAN (see man gcc) --- # when using SANitizers, we need to pass the -fsanitize.. # to both CFLAGS and LDFLAGS. The CFLAGS/LDFLAGS must be # specified as first in the list or there will be runtime # issues (for example user has to LD_PRELOAD asan for it to work # properly). if test -n "${SANITIZERS}"; then SANITIZERS=$(echo $SANITIZERS | sed -e 's/,/ /g') for SANITIZER in $SANITIZERS; do case $SANITIZER in asan|ASAN) SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=address" SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=address -lasan" AC_CHECK_LIB([asan],[main],,AC_MSG_ERROR([Unable to find libasan])) ;; ubsan|UBSAN) SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=undefined" SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=undefined -lubsan" AC_CHECK_LIB([ubsan],[main],,AC_MSG_ERROR([Unable to find libubsan])) ;; tsan|TSAN) SANITIZERS_CFLAGS="$SANITIZERS_CFLAGS -fsanitize=thread" SANITIZERS_LDFLAGS="$SANITIZERS_LDFLAGS -fsanitize=thread -ltsan" AC_CHECK_LIB([tsan],[main],,AC_MSG_ERROR([Unable to find libtsan])) ;; esac done fi DEFAULT_CFLAGS="-Werror -Wall -Wextra" # manual overrides # generates too much noise for stub APIs UNWANTED_CFLAGS="-Wno-unused-parameter -Wno-gnu-folding-constant" # check for character index warning AC_CACHE_CHECK([if -Wchar-subscripts warns for int8_t], [ac_cv_int8_t_warn], [ saveCFLAGS="$CFLAGS" CFLAGS="-Werror -Wchar-subscripts" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include ], [ char arr[256]; uint8_t i = 200; arr[i] = 'J'; return 0; ])], [ac_cv_int8_t_warn=no], [ac_cv_int8_t_warn=yes]) CFLAGS="$saveCFLAGS" ]) if test "x$ac_cv_int8_t_warn" = xyes; then UNWANTED_CFLAGS="$UNWANTED_CFLAGS -Wno-char-subscripts" fi AC_SUBST([AM_CFLAGS],["$SANITIZERS_CFLAGS $OPT_CFLAGS $GDB_CFLAGS $DEFAULT_CFLAGS $EXTRA_HARDENING_CFLAGS $UNWANTED_CFLAGS"]) LDFLAGS="$SANITIZERS_LDFLAGS $LDFLAGS" AC_SUBST([AM_LDFLAGS]) AC_SUBST([RUST_FLAGS]) AC_SUBST([RUST_TARGET_DIR]) AX_PROG_DATE AS_IF([test "$ax_cv_prog_date_gnu_date:$ax_cv_prog_date_gnu_utc" = yes:yes], [UTC_DATE_AT="date -u -d@"], [AS_IF([test "x$ax_cv_prog_date_bsd_date" = xyes], [UTC_DATE_AT="date -u -r"], [AC_MSG_ERROR([date utility unable to convert epoch to UTC])])]) AC_SUBST([UTC_DATE_AT]) AC_ARG_VAR([SOURCE_EPOCH],[last modification date of the source]) AC_MSG_NOTICE([trying to determine source epoch]) AC_MSG_CHECKING([for source epoch in \$SOURCE_EPOCH]) AS_IF([test -n "$SOURCE_EPOCH"], [AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no]) AC_MSG_CHECKING([for source epoch in source_epoch file]) AS_IF([test -e "$srcdir/source_epoch"], [read SOURCE_EPOCH <"$srcdir/source_epoch" AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no]) AC_MSG_CHECKING([for source epoch baked in by gitattributes export-subst]) SOURCE_EPOCH='$Format:%at$' # template for rewriting by git-archive AS_CASE([$SOURCE_EPOCH], [?Format:*], # was not rewritten [AC_MSG_RESULT([no]) AC_MSG_CHECKING([for source epoch in \$SOURCE_DATE_EPOCH]) AS_IF([test "x$SOURCE_DATE_EPOCH" != x], [SOURCE_EPOCH="$SOURCE_DATE_EPOCH" AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no]) AC_MSG_CHECKING([whether git log can provide a source epoch]) SOURCE_EPOCH=f${SOURCE_EPOCH#\$F} # convert into git log --pretty format SOURCE_EPOCH=$(cd "$srcdir" && git log -1 --pretty=${SOURCE_EPOCH%$} 2>/dev/null) AS_IF([test -n "$SOURCE_EPOCH"], [AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no, using current time and breaking reproducibility]) SOURCE_EPOCH=$(date +%s)])])], [AC_MSG_RESULT([yes])] )]) ]) AC_MSG_NOTICE([using source epoch $($UTC_DATE_AT$SOURCE_EPOCH +'%F %T %Z')]) AC_CONFIG_FILES([ Makefile libnozzle/Makefile libnozzle/libnozzle.pc libnozzle/tests/Makefile libnozzle/bindings/Makefile libnozzle/bindings/rust/Makefile libnozzle/bindings/rust/Cargo.toml libnozzle/bindings/rust/tests/Makefile libnozzle/bindings/rust/tests/Cargo.toml libknet/Makefile libknet/libknet.pc libknet/tests/Makefile libknet/bindings/Makefile libknet/bindings/rust/Makefile libknet/bindings/rust/Cargo.toml libknet/bindings/rust/tests/Makefile libknet/bindings/rust/tests/Cargo.toml man/Makefile man/Doxyfile-knet man/Doxyfile-nozzle ]) if test "x$VERSION" = "xUNKNOWN"; then AC_MSG_ERROR([m4_text_wrap([ configure was unable to determine the source tree's current version. This generally happens when using git archive (or the github download button) generated tarball/zip file. In order to workaround this issue, either use git clone https://github.com/kronosnet/kronosnet.git or use an official release tarball, available at https://kronosnet.org/releases/. Alternatively you can add a compatible version in a .tarball-version file at the top of the source tree, wipe your autom4te.cache dir and generated configure, and rerun autogen.sh. ], [ ], [ ], [76])]) fi AC_OUTPUT diff --git a/kronosnet.spec.in b/kronosnet.spec.in index 4d6b3358..afd40bcf 100644 --- a/kronosnet.spec.in +++ b/kronosnet.spec.in @@ -1,483 +1,474 @@ ############################################################################### ############################################################################### ## ## Copyright (C) 2012-2025 Red Hat, Inc. All rights reserved. ## ## This copyrighted material is made available to anyone wishing to use, ## modify, copy, or redistribute it subject to the terms and conditions ## of the GNU General Public License v.2 or higher ## ############################################################################### ############################################################################### # keep around ready for later user %global alphatag @alphatag@ %global numcomm @numcomm@ %global dirty @dirty@ # set defaults from ./configure invokation -%@sctp@ sctp %@nss@ nss %@openssl@ openssl %@gcrypt@ gcrypt %@zlib@ zlib %@lz4@ lz4 %@lzo2@ lzo2 %@lzma@ lzma %@bzip2@ bzip2 %@zstd@ zstd %@libnozzle@ libnozzle %@runautogen@ runautogen %@rpmdebuginfo@ rpmdebuginfo %@overriderpmdebuginfo@ overriderpmdebuginfo %@buildman@ buildman %@installtests@ installtests %if %{with overriderpmdebuginfo} %undefine _enable_debug_packages %endif # main (empty) package # http://www.rpm.org/max-rpm/s1-rpm-subpack-spec-file-changes.html Name: kronosnet Summary: Multipoint-to-Multipoint VPN daemon Version: @version@ Release: 1%{?numcomm:.%{numcomm}}%{?alphatag:.%{alphatag}}%{?dirty:.%{dirty}}%{?dist} License: GPL-2.0-or-later AND LGPL-2.1-or-later URL: https://kronosnet.org Source0: https://kronosnet.org/releases/%{name}-%{version}%{?numcomm:.%{numcomm}}%{?alphatag:-%{alphatag}}%{?dirty:-%{dirty}}.tar.gz # Build dependencies BuildRequires: gcc libqb-devel make # required to build man pages %if %{with buildman} BuildRequires: doxygen doxygen2man %endif -%if %{with sctp} -BuildRequires: lksctp-tools-devel -%endif %if %{with nss} %if 0%{?suse_version} BuildRequires: mozilla-nss-devel %else BuildRequires: nss-devel %endif %endif %if %{with openssl} %if 0%{?suse_version} BuildRequires: libopenssl-devel %else BuildRequires: openssl-devel %endif %endif %if %{with gcrypt} BuildRequires: libgcrypt-devel >= 1.8.0 %endif %if %{with zlib} BuildRequires: zlib-devel %endif %if %{with lz4} %if 0%{?suse_version} BuildRequires: liblz4-devel %else BuildRequires: lz4-devel %endif %endif %if %{with lzo2} BuildRequires: lzo-devel %endif %if %{with lzma} BuildRequires: xz-devel %endif %if %{with bzip2} %if 0%{?suse_version} BuildRequires: libbz2-devel %else BuildRequires: bzip2-devel %endif %endif %if %{with zstd} BuildRequires: libzstd-devel %endif %if %{with libnozzle} BuildRequires: libnl3-devel %endif %if %{with runautogen} BuildRequires: autoconf automake libtool %endif %prep %setup -q -n %{name}-%{version}%{?numcomm:.%{numcomm}}%{?alphatag:-%{alphatag}}%{?dirty:-%{dirty}} %build %if %{with runautogen} ./autogen.sh %endif %{configure} \ %if %{with installtests} --enable-install-tests \ %else --disable-install-tests \ %endif %if %{with buildman} --enable-man \ %else --disable-man \ %endif -%if %{with sctp} - --enable-libknet-sctp \ -%else - --disable-libknet-sctp \ -%endif %if %{with nss} --enable-crypto-nss \ %else --disable-crypto-nss \ %endif %if %{with openssl} --enable-crypto-openssl \ %else --disable-crypto-openssl \ %endif %if %{with gcrypt} --enable-crypto-gcrypt \ %else --disable-crypto-gcrypt \ %endif %if %{with zlib} --enable-compress-zlib \ %else --disable-compress-zlib \ %endif %if %{with lz4} --enable-compress-lz4 \ %else --disable-compress-lz4 \ %endif %if %{with lzo2} --enable-compress-lzo2 \ %else --disable-compress-lzo2 \ %endif %if %{with lzma} --enable-compress-lzma \ %else --disable-compress-lzma \ %endif %if %{with bzip2} --enable-compress-bzip2 \ %else --disable-compress-bzip2 \ %endif %if %{with zstd} --enable-compress-zstd \ %else --disable-compress-zstd \ %endif %if %{with libnozzle} --enable-libnozzle %else --disable-libnozzle %endif make %{_smp_mflags} %install rm -rf %{buildroot} make install DESTDIR=%{buildroot} # tree cleanup # remove static libraries find %{buildroot} -name "*.a" -exec rm {} \; # remove libtools leftovers find %{buildroot} -name "*.la" -exec rm {} \; # remove docs rm -rf %{buildroot}/usr/share/doc/kronosnet # main empty package %description The kronosnet source %if %{with libnozzle} %package -n libnozzle1 Summary: Simple userland wrapper around kernel tap devices License: LGPL-2.1-or-later %description -n libnozzle1 This is an over-engineered commodity library to manage a pool of tap devices and provides the basic pre-up.d/up.d/down.d/post-down.d infrastructure. %files -n libnozzle1 %license COPYING.* COPYRIGHT %{_libdir}/libnozzle.so.* %if 0%{?ldconfig_scriptlets} %ldconfig_scriptlets -n libnozzle1 %else %post -n libnozzle1 -p /sbin/ldconfig %postun -n libnozzle1 -p /sbin/ldconfig %endif %package -n libnozzle1-devel Summary: Simple userland wrapper around kernel tap devices (developer files) License: LGPL-2.1-or-later Requires: libnozzle1%{_isa} = %{version}-%{release} Requires: pkgconfig %description -n libnozzle1-devel This is an over-engineered commodity library to manage a pool of tap devices and provides the basic pre-up.d/up.d/down.d/post-down.d infrastructure. %files -n libnozzle1-devel %license COPYING.* COPYRIGHT %{_libdir}/libnozzle.so %{_includedir}/libnozzle.h %{_libdir}/pkgconfig/libnozzle.pc %if %{with buildman} %{_mandir}/man3/nozzle*.3.gz %endif %endif %package -n libknet1 Summary: Kronosnet core switching implementation License: LGPL-2.1-or-later %description -n libknet1 The whole kronosnet core is implemented in this library. Please refer to the not-yet-existing documentation for further information. %files -n libknet1 %license COPYING.* COPYRIGHT %{_libdir}/libknet.so.* %dir %{_libdir}/kronosnet %if 0%{?ldconfig_scriptlets} %ldconfig_scriptlets -n libknet1 %else %post -n libknet1 -p /sbin/ldconfig %postun -n libknet1 -p /sbin/ldconfig %endif %package -n libknet1-devel Summary: Kronosnet core switching implementation (developer files) License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} Requires: pkgconfig %description -n libknet1-devel The whole kronosnet core is implemented in this library. Please refer to the not-yet-existing documentation for further information. %files -n libknet1-devel %license COPYING.* COPYRIGHT %{_libdir}/libknet.so %{_includedir}/libknet.h %{_libdir}/pkgconfig/libknet.pc %if %{with buildman} %{_mandir}/man3/knet*.3.gz %endif %if %{with nss} %package -n libknet1-crypto-nss-plugin Summary: Provides libknet1 nss support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-crypto-nss-plugin Provides NSS crypto support for libknet1. %files -n libknet1-crypto-nss-plugin %{_libdir}/kronosnet/crypto_nss.so %endif %if %{with openssl} %package -n libknet1-crypto-openssl-plugin Summary: Provides libknet1 openssl support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-crypto-openssl-plugin Provides OpenSSL crypto support for libknet1. %files -n libknet1-crypto-openssl-plugin %{_libdir}/kronosnet/crypto_openssl.so %endif %if %{with gcrypt} %package -n libknet1-crypto-gcrypt-plugin Summary: Provides libknet1 gcrypt support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-crypto-gcrypt-plugin Provides libgcrypt crypto support for libknet1. %files -n libknet1-crypto-gcrypt-plugin %{_libdir}/kronosnet/crypto_gcrypt.so %endif %if %{with zlib} %package -n libknet1-compress-zlib-plugin Summary: Provides libknet1 zlib support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-compress-zlib-plugin Provides zlib compression support for libknet1. %files -n libknet1-compress-zlib-plugin %{_libdir}/kronosnet/compress_zlib.so %endif %if %{with lz4} %package -n libknet1-compress-lz4-plugin Summary: Provides libknet1 lz4 and lz4hc support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-compress-lz4-plugin Provides lz4 and lz4hc compression support for libknet1. %files -n libknet1-compress-lz4-plugin %{_libdir}/kronosnet/compress_lz4.so %{_libdir}/kronosnet/compress_lz4hc.so %endif %if %{with lzo2} %package -n libknet1-compress-lzo2-plugin Summary: Provides libknet1 lzo2 support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-compress-lzo2-plugin Provides lzo2 compression support for libknet1. %files -n libknet1-compress-lzo2-plugin %{_libdir}/kronosnet/compress_lzo2.so %endif %if %{with lzma} %package -n libknet1-compress-lzma-plugin Summary: Provides libknet1 lzma support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-compress-lzma-plugin Provides lzma compression support for libknet1. %files -n libknet1-compress-lzma-plugin %{_libdir}/kronosnet/compress_lzma.so %endif %if %{with bzip2} %package -n libknet1-compress-bzip2-plugin Summary: Provides libknet1 bzip2 support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-compress-bzip2-plugin Provides bzip2 compression support for libknet1. %files -n libknet1-compress-bzip2-plugin %{_libdir}/kronosnet/compress_bzip2.so %endif %if %{with zstd} %package -n libknet1-compress-zstd-plugin Summary: Provides libknet1 zstd support License: LGPL-2.1-or-later Requires: libknet1%{_isa} = %{version}-%{release} %description -n libknet1-compress-zstd-plugin Provides zstd compression support for libknet1. %files -n libknet1-compress-zstd-plugin %{_libdir}/kronosnet/compress_zstd.so %endif %package -n libknet1-crypto-plugins-all Summary: Provides libknet1 crypto plugins meta package License: LGPL-2.1-or-later %if %{with nss} Requires: libknet1-crypto-nss-plugin%{_isa} = %{version}-%{release} %endif %if %{with openssl} Requires: libknet1-crypto-openssl-plugin%{_isa} = %{version}-%{release} %endif %if %{with gcrypt} Requires: libknet1-crypto-gcrypt-plugin%{_isa} = %{version}-%{release} %endif %description -n libknet1-crypto-plugins-all Provides meta package to install all of libknet1 crypto plugins %files -n libknet1-crypto-plugins-all %package -n libknet1-compress-plugins-all Summary: Provides libknet1 compress plugins meta package License: LGPL-2.1-or-later %if %{with zlib} Requires: libknet1-compress-zlib-plugin%{_isa} = %{version}-%{release} %endif %if %{with lz4} Requires: libknet1-compress-lz4-plugin%{_isa} = %{version}-%{release} %endif %if %{with lzo2} Requires: libknet1-compress-lzo2-plugin%{_isa} = %{version}-%{release} %endif %if %{with lzma} Requires: libknet1-compress-lzma-plugin%{_isa} = %{version}-%{release} %endif %if %{with bzip2} Requires: libknet1-compress-bzip2-plugin%{_isa} = %{version}-%{release} %endif %if %{with zstd} Requires: libknet1-compress-zstd-plugin%{_isa} = %{version}-%{release} %endif %description -n libknet1-compress-plugins-all Meta package to install all of libknet1 compress plugins %files -n libknet1-compress-plugins-all %package -n libknet1-plugins-all Summary: Provides libknet1 plugins meta package License: LGPL-2.1-or-later Requires: libknet1-compress-plugins-all%{_isa} = %{version}-%{release} Requires: libknet1-crypto-plugins-all%{_isa} = %{version}-%{release} %description -n libknet1-plugins-all Meta package to install all of libknet1 plugins %files -n libknet1-plugins-all %if %{with installtests} %package -n kronosnet-tests Summary: Provides kronosnet test suite License: GPL-2.0-or-later Requires: libknet1%{_isa} = %{version}-%{release} %if %{with libnozzle} Requires: libnozzle1%{_isa} = %{version}-%{release} %endif %description -n kronosnet-tests This package contains all the libknet and libnozzle test suite. %files -n kronosnet-tests %{_libdir}/kronosnet/tests/* %endif %if %{with rpmdebuginfo} %debug_package %endif %changelog * @date@ Autotools generated version - @version@-1-@numcomm@.@alphatag@.@dirty@ - These aren't the droids you're looking for. diff --git a/libknet/Makefile.am b/libknet/Makefile.am index 598de296..f869fb7e 100644 --- a/libknet/Makefile.am +++ b/libknet/Makefile.am @@ -1,182 +1,180 @@ # # Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. # # Authors: Fabio M. Di Nitto # Federico Simoncelli # # This software licensed under GPL-2.0+ # MAINTAINERCLEANFILES = Makefile.in include $(top_srcdir)/build-aux/check.mk SYMFILE = libknet_exported_syms EXTRA_DIST = $(SYMFILE) SUBDIRS = . tests bindings # override global LIBS that pulls in lots of craft we don't need here LIBS = sources = \ common.c \ compat.c \ compress.c \ crypto.c \ handle.c \ handle_api.c \ host.c \ lib_config.c \ links.c \ links_acl.c \ links_acl_ip.c \ links_acl_loopback.c \ logging.c \ netutils.c \ onwire.c \ onwire_v1.c \ threads_common.c \ threads_dsthandler.c \ threads_heartbeat.c \ threads_pmtud.c \ threads_rx.c \ threads_tx.c \ transports.c \ transport_common.c \ transport_loopback.c \ - transport_udp.c \ - transport_sctp.c + transport_udp.c include_HEADERS = libknet.h pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = libknet.pc noinst_HEADERS = \ common.h \ compat.h \ compress.h \ compress_model.h \ crypto.h \ crypto_model.h \ host.h \ internals.h \ links.h \ links_acl.h \ links_acl_ip.h \ links_acl_loopback.h \ logging.h \ netutils.h \ onwire.h \ onwire_v1.h \ threads_common.h \ threads_dsthandler.h \ threads_heartbeat.h \ threads_pmtud.h \ threads_rx.h \ threads_tx.h \ transports.h \ transport_common.h \ transport_loopback.h \ - transport_udp.h \ - transport_sctp.h + transport_udp.h lib_LTLIBRARIES = libknet.la libknet_la_SOURCES = $(sources) AM_CFLAGS += $(XOPEN_CPPFLAGS) $(libqb_CFLAGS) libknet_la_CFLAGS = $(AM_CFLAGS) $(PTHREAD_CFLAGS) $(zlib_CFLAGS) \ -DPLUGINPATH="\"$(pkglibdir)\"" EXTRA_libknet_la_DEPENDENCIES = $(SYMFILE) if BUILD_FOR_SOLARIS libknet_la_LDFLAGS = $(AM_LDFLAGS) \ -Wl,-M$(abs_srcdir)/$(SYMFILE) \ -version-info $(libknetversion) else libknet_la_LDFLAGS = $(AM_LDFLAGS) \ -Wl,--version-script=$(abs_srcdir)/$(SYMFILE) \ -version-info $(libknetversion) endif libknet_la_LIBADD = $(PTHREAD_LIBS) $(socket_LIBS) $(dl_LIBS) $(rt_LIBS) $(m_LIBS) $(zlib_LIBS) check-local: check-annocheck-libs # Prepare empty value for appending pkglib_LTLIBRARIES = # MODULE_LDFLAGS would mean a target-specific variable for Automake MODULELDFLAGS = $(AM_LDFLAGS) -module -avoid-version -export-dynamic if BUILD_COMPRESS_ZSTD pkglib_LTLIBRARIES += compress_zstd.la compress_zstd_la_LDFLAGS = $(MODULELDFLAGS) compress_zstd_la_CFLAGS = $(AM_CFLAGS) $(libzstd_CFLAGS) compress_zstd_la_LIBADD = $(libzstd_LIBS) endif if BUILD_COMPRESS_ZLIB pkglib_LTLIBRARIES += compress_zlib.la compress_zlib_la_LDFLAGS = $(MODULELDFLAGS) compress_zlib_la_CFLAGS = $(AM_CFLAGS) $(zlib_CFLAGS) compress_zlib_la_LIBADD = $(zlib_LIBS) endif if BUILD_COMPRESS_LZ4 pkglib_LTLIBRARIES += compress_lz4.la compress_lz4hc.la compress_lz4_la_LDFLAGS = $(MODULELDFLAGS) compress_lz4_la_CFLAGS = $(AM_CFLAGS) $(liblz4_CFLAGS) compress_lz4_la_LIBADD = $(liblz4_LIBS) compress_lz4hc_la_LDFLAGS = $(MODULELDFLAGS) compress_lz4hc_la_CFLAGS = $(AM_CFLAGS) $(liblz4_CFLAGS) compress_lz4hc_la_LIBADD = $(liblz4_LIBS) endif if BUILD_COMPRESS_LZO2 pkglib_LTLIBRARIES += compress_lzo2.la compress_lzo2_la_LDFLAGS = $(MODULELDFLAGS) compress_lzo2_la_CFLAGS = $(AM_CFLAGS) $(lzo2_CFLAGS) compress_lzo2_la_LIBADD = $(lzo2_LIBS) endif if BUILD_COMPRESS_LZMA pkglib_LTLIBRARIES += compress_lzma.la compress_lzma_la_LDFLAGS = $(MODULELDFLAGS) compress_lzma_la_CFLAGS = $(AM_CFLAGS) $(liblzma_CFLAGS) compress_lzma_la_LIBADD = $(liblzma_LIBS) endif if BUILD_COMPRESS_BZIP2 pkglib_LTLIBRARIES += compress_bzip2.la compress_bzip2_la_LDFLAGS = $(MODULELDFLAGS) compress_bzip2_la_CFLAGS = $(AM_CFLAGS) $(bzip2_CFLAGS) compress_bzip2_la_LIBADD = $(bzip2_LIBS) endif if BUILD_CRYPTO_NSS pkglib_LTLIBRARIES += crypto_nss.la crypto_nss_la_LDFLAGS = $(MODULELDFLAGS) crypto_nss_la_CFLAGS = $(AM_CFLAGS) $(nss_CFLAGS) crypto_nss_la_LIBADD = $(nss_LIBS) endif if BUILD_CRYPTO_OPENSSL pkglib_LTLIBRARIES += crypto_openssl.la crypto_openssl_la_LDFLAGS = $(MODULELDFLAGS) crypto_openssl_la_CFLAGS = $(AM_CFLAGS) $(openssl_CFLAGS) crypto_openssl_la_LIBADD = $(openssl_LIBS) endif if BUILD_CRYPTO_GCRYPT pkglib_LTLIBRARIES += crypto_gcrypt.la crypto_gcrypt_la_LDFLAGS = $(MODULELDFLAGS) crypto_gcrypt_la_CFLAGS = $(AM_CFLAGS) $(gcrypt_CFLAGS) crypto_gcrypt_la_LIBADD = $(gcrypt_LIBS) endif diff --git a/libknet/bindings/rust/src/knet_bindings.rs b/libknet/bindings/rust/src/knet_bindings.rs index 3ab04197..6b24c42d 100644 --- a/libknet/bindings/rust/src/knet_bindings.rs +++ b/libknet/bindings/rust/src/knet_bindings.rs @@ -1,2619 +1,2612 @@ // libknet interface for Rust // Copyright (C) 2021-2025 Red Hat, Inc. // // All rights reserved. // // Author: Christine Caulfield (ccaulfi@redhat.com) // #![allow(clippy::too_many_arguments)] #![allow(clippy::collapsible_else_if)] #![allow(clippy::bad_bit_mask)] // For the code generated by bindgen use crate::sys::libknet as ffi; use std::ffi::{CString, CStr}; use std::sync::mpsc::*; use std::ptr::{copy_nonoverlapping, null, null_mut}; use std::sync::Mutex; use std::collections::HashMap; use std::io::{Result, Error, ErrorKind}; use std::os::raw::{c_void, c_char, c_uchar, c_uint}; use std::mem::size_of; use std::net::SocketAddr; use std::fmt; use std::thread::spawn; use std::time::{Duration, SystemTime}; use os_socketaddr::OsSocketAddr; #[derive(Copy, Clone, PartialEq, Eq)] /// The ID of a host known to knet. pub struct HostId { host_id: u16, } impl HostId { pub fn new(id: u16) -> HostId { HostId{host_id: id} } pub fn to_u16(self: HostId) -> u16 { self.host_id } } impl fmt::Display for HostId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f,"{}", self.host_id)?; Ok(()) } } pub enum TxRx { Tx = 0, Rx = 1 } impl TxRx { pub fn new (tx_rx: u8) -> TxRx { match tx_rx { 1 => TxRx::Rx, _ => TxRx::Tx } } } impl fmt::Display for TxRx { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { TxRx::Tx => write!(f, "Tx"), TxRx::Rx => write!(f, "Rx"), } } } bitflags! { /// Flags passed into [handle_new] pub struct HandleFlags: u64 { const PRIVILEGED = 1; const NONE = 0; } } bitflags! { /// Flags passed into [handle_add_datafd] pub struct DataFdFlags: u32 { const NONE = 0; const RX_RETURN_INFO = 1; } } bitflags! { /// Flags passed into [link_set_config] pub struct LinkFlags: u64 { const TRAFFICHIPRIO = 1; const NONE = 0; } } /// for passing to [handle_crypto_set_config] pub struct CryptoConfig<'a> { pub crypto_model: String, pub crypto_cipher_type: String, pub crypto_hash_type: String, pub private_key: &'a [u8], } /// for passing to [handle_compress] pub struct CompressConfig { pub compress_model: String, pub compress_threshold: u32, pub compress_level: i32, } /// Return value from packet filter pub enum FilterDecision { Discard, Unicast, Multicast } impl FilterDecision { pub fn to_i32(self: &FilterDecision) -> i32 { match self { FilterDecision::Discard => -1, FilterDecision::Unicast => 0, FilterDecision::Multicast => 1, } } } impl fmt::Display for FilterDecision { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { FilterDecision::Discard => write!(f, "Discard"), FilterDecision::Unicast => write!(f, "Unicast"), FilterDecision::Multicast => write!(f, "Multicast"), } } } // Used to convert a knet_handle_t into one of ours lazy_static! { static ref HANDLE_HASH: Mutex> = Mutex::new(HashMap::new()); } fn get_errno() -> i32 { match Error::last_os_error().raw_os_error() { Some(e) => e, None => libc::EINVAL, } } /// Callback from [handle_enable_sock_notify] pub type SockNotifyFn = fn(private_data: u64, datafd: i32, channel: i8, txrx: TxRx, Result<()>); /// Callback called when packets arrive/are sent [handle_enable_filter] pub type FilterFn = fn(private_data: u64, outdata: &[u8], txrx: TxRx, this_host_id: HostId, src_host_id: HostId, channel: &mut i8, dst_host_ids: &mut Vec) -> FilterDecision; /// Callback called when PMTU changes, see [handle_enable_pmtud_notify] pub type PmtudNotifyFn = fn(private_data: u64, data_mtu: u32); /// Called when the onwire version number for a node changes, see [handle_enable_onwire_ver_notify] pub type OnwireNotifyFn = fn(private_data: u64, onwire_min_ver: u8, onwire_max_ver: u8, onwire_ver: u8); /// Called when a host status changes, see [host_enable_status_change_notify] pub type HostStatusChangeNotifyFn = fn(private_data: u64, host_id: HostId, reachable: bool, remote: bool, external: bool); /// Called when a link status changes, see [link_enable_status_change_notify] pub type LinkStatusChangeNotifyFn = fn(private_data: u64, host_id: HostId, link_id: u8, connected: bool, remote: bool, external: bool); // Called from knet, we work out where to route it to and convert params extern "C" fn rust_sock_notify_fn( private_data: *mut c_void, datafd: i32, channel: i8, tx_rx: u8, error: i32, errorno: i32) { if let Some(h) = HANDLE_HASH.lock().unwrap().get(&(private_data as u64)) { let res = if error == 0 { Ok(()) } else { Err(Error::from_raw_os_error(errorno)) }; // Call user fn if let Some(f) = h.sock_notify_fn { f(h.sock_notify_private_data, datafd, channel, TxRx::new(tx_rx), res); } } } // Called from knet, we work out where to route it to and convert params extern "C" fn rust_filter_fn( private_data: *mut c_void, outdata: *const c_uchar, outdata_len: isize, tx_rx: u8, this_host_id: u16, src_host_id: u16, channel: *mut i8, dst_host_ids: *mut u16, dst_host_ids_entries: *mut usize) -> i32 { let mut ret : i32 = -1; if let Some(h) = HANDLE_HASH.lock().unwrap().get(&(private_data as u64)) { // Is there is filter fn? if let Some(f) = h.filter_fn { let data : &[u8] = unsafe { std::slice::from_raw_parts(outdata, outdata_len as usize) }; let mut r_channel = unsafe {*channel}; let mut hosts_vec = Vec::::new(); // Call Rust callback ret = f(h.filter_private_data, data, TxRx::new(tx_rx), HostId{host_id: this_host_id}, HostId{host_id: src_host_id}, &mut r_channel, &mut hosts_vec).to_i32(); // Pass back mutable params dst_hosts unsafe { *channel = r_channel; *dst_host_ids_entries = hosts_vec.len(); let mut retvec = dst_host_ids; for i in &hosts_vec { *retvec = i.host_id; retvec = retvec.offset(1); // next entry } } } } ret } // Called from knet, we work out where to route it to and convert params extern "C" fn rust_pmtud_notify_fn( private_data: *mut c_void, data_mtu: u32) { if let Some(h) = HANDLE_HASH.lock().unwrap().get(&(private_data as u64)) { // Call user fn if let Some(f) = h.pmtud_notify_fn { f(h.pmtud_notify_private_data, data_mtu); } } } // Called from knet, we work out where to route it to and convert params extern "C" fn rust_onwire_notify_fn( private_data: *mut c_void, onwire_min_ver: u8, onwire_max_ver: u8, onwire_ver: u8) { if let Some(h) = HANDLE_HASH.lock().unwrap().get(&(private_data as u64)) { // Call user fn if let Some(f) = h.onwire_notify_fn { f(h.onwire_notify_private_data, onwire_min_ver, onwire_max_ver, onwire_ver); } } } // Called from knet, we work out where to route it to and convert params extern "C" fn rust_host_status_change_notify_fn( private_data: *mut c_void, host_id: u16, reachable: u8, remote: u8, external: u8) { if let Some(h) = HANDLE_HASH.lock().unwrap().get(&(private_data as u64)) { // Call user fn if let Some(f) = h.host_status_change_notify_fn { f(h.host_status_change_notify_private_data, HostId{host_id}, crate::u8_to_bool(reachable), crate::u8_to_bool(remote), crate::u8_to_bool(external)); } } } // Called from knet, we work out where to route it to and convert params extern "C" fn rust_link_status_change_notify_fn( private_data: *mut c_void, host_id: u16, link_id: u8, connected: u8, remote: u8, external: u8) { if let Some(h) = HANDLE_HASH.lock().unwrap().get(&(private_data as u64)) { // Call user fn if let Some(f) = h.link_status_change_notify_fn { f(h.link_status_change_notify_private_data, HostId{host_id}, link_id, crate::u8_to_bool(connected), crate::u8_to_bool(remote), crate::u8_to_bool(external)); } } } // Logging thread fn logging_thread(knet_pipe: i32, sender: Sender) { let mut logbuf = ffi::knet_log_msg {msg: [0; 254], subsystem: 0, msglevel: 0, knet_h: 0 as ffi::knet_handle_t}; // Make it blocking unsafe { libc::fcntl(knet_pipe, libc::F_SETFL, libc::fcntl(knet_pipe, libc::F_GETFL, 0) & !libc::O_NONBLOCK)}; loop { let msglen = unsafe {libc::read(knet_pipe, &mut logbuf as *mut _ as *mut c_void, size_of::())}; if msglen < 1 { unsafe { libc::close(knet_pipe); } // EOF on pipe, handle is closed. return; } if msglen == size_of::() as isize { let rmsg = LogMsg { msg: crate::string_from_bytes_safe(logbuf.msg.as_ptr(), 254), subsystem: SubSystem::new(logbuf.subsystem), level: LogLevel::new(logbuf.msglevel), handle: Handle{knet_handle: logbuf.knet_h as u64, clone: true}}; if let Err(e) = sender.send(rmsg) { println!("Error sending log message: {e}"); } } } } /// a handle into the knet library, returned from [handle_new] pub struct Handle { pub knet_handle: u64, clone: bool, // clone Handles don't trigger knet_handle_free() } impl Clone for Handle { fn clone(&self) -> Handle { Handle {knet_handle: self.knet_handle, clone: true} } } // Clones count as equivalent impl PartialEq for Handle { fn eq(&self, other: &Handle) -> bool { self.knet_handle == other.knet_handle } } // Private version of knet handle, contains all the callback data so // we only need to access it in the calback functions, making the rest // a bit quicker & neater struct PrivHandle { log_fd: i32, sock_notify_fn: Option, sock_notify_private_data: u64, filter_fn: Option, filter_private_data: u64, pmtud_notify_fn: Option, pmtud_notify_private_data: u64, onwire_notify_fn: Option, onwire_notify_private_data: u64, host_status_change_notify_fn: Option, host_status_change_notify_private_data: u64, link_status_change_notify_fn: Option, link_status_change_notify_private_data: u64, } /// A knet logging message returned down the log_sender channel set in [handle_new] pub struct LogMsg { pub msg: String, pub subsystem: SubSystem, pub level: LogLevel, pub handle: Handle, } /// Initialise the knet library, returns a handle for use with the other API calls pub fn handle_new(host_id: &HostId, log_sender: Option>, default_log_level: LogLevel, flags: HandleFlags) -> Result { // If a log sender was passed, make an FD & thread for knet let log_fd = match log_sender { Some(s) => { let mut pipes = [0i32; 2]; if unsafe {libc::pipe(pipes.as_mut_ptr())} != 0 { return Err(Error::last_os_error()); } spawn(move || logging_thread(pipes[0], s)); pipes[1] }, None => 0 }; let res = unsafe { ffi::knet_handle_new(host_id.host_id, log_fd, default_log_level.to_u8(), flags.bits()) }; if res.is_null() { Err(Error::last_os_error()) } else { let rhandle = PrivHandle{log_fd, sock_notify_fn: None, sock_notify_private_data: 0u64, filter_fn: None, filter_private_data: 0u64, pmtud_notify_fn: None, pmtud_notify_private_data: 0u64, onwire_notify_fn: None, onwire_notify_private_data: 0u64, host_status_change_notify_fn: None, host_status_change_notify_private_data: 0u64, link_status_change_notify_fn: None, link_status_change_notify_private_data: 0u64, }; HANDLE_HASH.lock().unwrap().insert(res as u64, rhandle); Ok(Handle{knet_handle: res as u64, clone: false}) } } /// Finish with knet, frees the handle returned by [handle_new] pub fn handle_free(handle: &Handle) -> Result<()> { if handle_free_knet_handle(handle.knet_handle) == 0 { Ok(()) } else { Err(Error::last_os_error()) } } fn handle_free_knet_handle(knet_handle: u64) -> i32 { let res = unsafe { ffi::knet_handle_free(knet_handle as ffi::knet_handle_t) }; if res == 0 { // Close the log fd as knet doesn't "do ownership" and this will shut down // our logging thread. if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(knet_handle)) { unsafe { libc::close(h.log_fd); }; } HANDLE_HASH.lock().unwrap().remove(&knet_handle); } res } impl Drop for Handle { fn drop(self: &mut Handle) { if !self.clone { handle_free_knet_handle(self.knet_handle); } } } /// Enable notifications of socket state changes, set callback to 'None' to disable pub fn handle_enable_sock_notify(handle: &Handle, private_data: u64, sock_notify_fn: Option) -> Result<()> { if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(handle.knet_handle)) { h.sock_notify_private_data = private_data; h.sock_notify_fn = sock_notify_fn; let res = match sock_notify_fn { Some(_f) => unsafe { ffi::knet_handle_enable_sock_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, Some(rust_sock_notify_fn)) }, None => unsafe { ffi::knet_handle_enable_sock_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, None) }, }; if res == 0 { return Ok(()); } else { return Err(Error::last_os_error()); } } Err(Error::other("Rust handle not found")) } /// Add a data FD to knet (with flags). if datafd is 0 then knet will allocate one for you. pub fn handle_add_datafd(handle: &Handle, datafd: i32, channel: i8, flags: DataFdFlags) -> Result<(i32, i8)> { let mut c_datafd = datafd; let mut c_channel = channel; let res = unsafe { ffi::knet_handle_add_datafd(handle.knet_handle as ffi::knet_handle_t, &mut c_datafd, &mut c_channel, flags.bits()) }; if res == 0 { Ok((c_datafd, c_channel)) } else { Err(Error::last_os_error()) } } /// Remove a datafd from knet pub fn handle_remove_datafd(handle: &Handle, datafd: i32) -> Result<()> { let res = unsafe { ffi::knet_handle_remove_datafd(handle.knet_handle as ffi::knet_handle_t, datafd) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Returns the channel associated with data fd pub fn handle_get_channel(handle: &Handle, datafd: i32) -> Result { let mut c_channel = 0i8; let res = unsafe { ffi::knet_handle_get_channel(handle.knet_handle as ffi::knet_handle_t, datafd, &mut c_channel) }; if res == 0 { Ok(c_channel) } else { Err(Error::last_os_error()) } } /// Returns the data FD associated with a channel pub fn handle_get_datafd(handle: &Handle, channel: i8) -> Result { let mut c_datafd = 0i32; let res = unsafe { ffi::knet_handle_get_datafd(handle.knet_handle as ffi::knet_handle_t, channel, &mut c_datafd) }; if res == 0 { Ok(c_datafd) } else { Err(Error::last_os_error()) } } #[derive(Copy, Clone, PartialEq, Eq)] pub enum DefragReclaimPolicy { Average = 0, Absolute = 1 } impl DefragReclaimPolicy { pub fn new (policy: u32) -> DefragReclaimPolicy { { match policy { 1 => DefragReclaimPolicy::Absolute, _ => DefragReclaimPolicy::Average, } } } pub fn to_u32 (&self) -> u32 { { match self { DefragReclaimPolicy::Absolute => 1, DefragReclaimPolicy::Average => 0, } } } } impl fmt::Display for DefragReclaimPolicy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { DefragReclaimPolicy::Absolute => write!(f, "Absolute"), DefragReclaimPolicy::Average => write!(f, "Average"), } } } /// Configure the defrag buffer parameters - applies to all hosts pub fn handle_set_host_defrag_bufs(handle: &Handle, min_defrag_bufs: u16, max_defrag_bufs: u16, shrink_threshold: u8, reclaim_policy: DefragReclaimPolicy) -> Result<()> { let res = unsafe { ffi::knet_handle_set_host_defrag_bufs(handle.knet_handle as ffi::knet_handle_t, min_defrag_bufs, max_defrag_bufs, shrink_threshold, reclaim_policy.to_u32()) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the defrag buffer parameters. /// Returns (min_defrag_bufs, max_defrag_bufs, shrink_threshold, usage_samples, usage_samples_timespan, reclaim_policy) pub fn handle_get_host_defrag_bufs(handle: &Handle) -> Result<(u16, u16, u8, DefragReclaimPolicy)> { let mut min_defrag_bufs: u16 = 0; let mut max_defrag_bufs: u16 = 0; let mut shrink_threshold: u8 = 0; let mut reclaim_policy: u32 = 0; let res = unsafe { ffi::knet_handle_get_host_defrag_bufs(handle.knet_handle as ffi::knet_handle_t, &mut min_defrag_bufs, &mut max_defrag_bufs, &mut shrink_threshold, &mut reclaim_policy) }; if res == 0 { Ok((min_defrag_bufs, max_defrag_bufs, shrink_threshold, DefragReclaimPolicy::new(reclaim_policy))) } else { Err(Error::last_os_error()) } } /// Receive messages from knet pub fn recv(handle: &Handle, buf: &[u8], channel: i8) -> Result { let res = unsafe { ffi::knet_recv(handle.knet_handle as ffi::knet_handle_t, buf.as_ptr() as *mut c_char, buf.len(), channel) }; if res >= 0 { Ok(res) } else { if get_errno() == libc::EAGAIN { Err(Error::new(ErrorKind::WouldBlock, "Try again")) } else { Err(Error::last_os_error()) } } } /// Send messages knet pub fn send(handle: &Handle, buf: &[u8], channel: i8) -> Result { let res = unsafe { ffi::knet_send(handle.knet_handle as ffi::knet_handle_t, buf.as_ptr() as *const c_char, buf.len(), channel) }; if res >= 0 { Ok(res) } else { if get_errno() == libc::EAGAIN { Err(Error::new(ErrorKind::WouldBlock, "Try again")) } else { Err(Error::last_os_error()) } } } /// Send messages to knet and wait till they have gone pub fn send_sync(handle: &Handle, buf: &[u8], channel: i8) -> Result<()> { let res = unsafe { ffi::knet_send_sync(handle.knet_handle as ffi::knet_handle_t, buf.as_ptr() as *const c_char, buf.len(), channel) }; if res == 0 { Ok(()) } else { if get_errno() == libc::EAGAIN { Err(Error::new(ErrorKind::WouldBlock, "Try again")) } else { Err(Error::last_os_error()) } } } /// Enable the packet filter. pass 'None' as the callback to disable. pub fn handle_enable_filter(handle: &Handle, private_data: u64, filter_fn: Option) -> Result<()> { if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(handle.knet_handle)) { h.filter_private_data = private_data; h.filter_fn = filter_fn; let res = match filter_fn { Some(_f) => unsafe { ffi::knet_handle_enable_filter(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, Some(rust_filter_fn)) }, None => unsafe { ffi::knet_handle_enable_filter(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, None) }, }; if res == 0 { return Ok(()); } else { return Err(Error::last_os_error()); } }; Err(Error::other("Rust handle not found")) } /// Set timer resolution pub fn handle_set_threads_timer_res(handle: &Handle, timeres: u32) -> Result<()> { let res = unsafe { ffi::knet_handle_set_threads_timer_res(handle.knet_handle as ffi::knet_handle_t, timeres) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get timer resolution pub fn handle_get_threads_timer_res(handle: &Handle) -> Result { let mut c_timeres: u32 = 0; let res = unsafe { ffi::knet_handle_get_threads_timer_res(handle.knet_handle as ffi::knet_handle_t, &mut c_timeres) }; if res == 0 { Ok(c_timeres) } else { Err(Error::last_os_error()) } } /// Starts traffic moving. You must call this before knet will do anything. pub fn handle_setfwd(handle: &Handle, enabled: bool) -> Result<()> { let res = unsafe { ffi::knet_handle_setfwd(handle.knet_handle as ffi::knet_handle_t, enabled as c_uint) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Enable access control lists pub fn handle_enable_access_lists(handle: &Handle, enabled: bool) -> Result<()> { let res = unsafe { ffi::knet_handle_enable_access_lists(handle.knet_handle as ffi::knet_handle_t, enabled as c_uint) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Set frequency that PMTUd will check for MTU changes. value in milliseconds pub fn handle_pmtud_setfreq(handle: &Handle, interval: u32) -> Result<()> { let res = unsafe { ffi::knet_handle_pmtud_setfreq(handle.knet_handle as ffi::knet_handle_t, interval) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get frequency that PMTUd will check for MTU changes. value in milliseconds pub fn handle_pmtud_getfreq(handle: &Handle) -> Result { let mut c_interval = 0u32; let res = unsafe { ffi::knet_handle_pmtud_getfreq(handle.knet_handle as ffi::knet_handle_t, &mut c_interval) }; if res == 0 { Ok(c_interval) } else { Err(Error::last_os_error()) } } /// Get the current MTU pub fn handle_pmtud_get(handle: &Handle) -> Result { let mut c_mtu = 0u32; let res = unsafe { ffi::knet_handle_pmtud_get(handle.knet_handle as ffi::knet_handle_t, &mut c_mtu) }; if res == 0 { Ok(c_mtu) } else { Err(Error::last_os_error()) } } /// Set the interface MTU (this should not be necessary) pub fn handle_pmtud_set(handle: &Handle, iface_mtu: u32) -> Result<()> { let res = unsafe { ffi::knet_handle_pmtud_set(handle.knet_handle as ffi::knet_handle_t, iface_mtu) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Enable notification of MTU changes pub fn handle_enable_pmtud_notify(handle: &Handle, private_data: u64, pmtud_notify_fn: Option) -> Result<()> { if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(handle.knet_handle)) { h.pmtud_notify_private_data = private_data; h.pmtud_notify_fn = pmtud_notify_fn; let res = match pmtud_notify_fn { Some(_f) => unsafe { ffi::knet_handle_enable_pmtud_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, Some(rust_pmtud_notify_fn)) }, None => unsafe { ffi::knet_handle_enable_pmtud_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, None) }, }; if res == 0 { return Ok(()); } else { return Err(Error::last_os_error()); } } Err(Error::other("Rust handle not found")) } /// Configure cryptographic seetings for packets being transmitted pub fn handle_crypto_set_config(handle: &Handle, config: &CryptoConfig, config_num: u8) -> Result<()> { let mut crypto_cfg = ffi::knet_handle_crypto_cfg { crypto_model: [0; 16], crypto_cipher_type: [0; 16], crypto_hash_type: [0; 16], private_key: [0; 4096], private_key_len: 0, }; if config.private_key.len() > 4096 { return Err(Error::other("key too long")); } crate::string_to_bytes(&config.crypto_model, &mut crypto_cfg.crypto_model)?; crate::string_to_bytes(&config.crypto_cipher_type, &mut crypto_cfg.crypto_cipher_type)?; crate::string_to_bytes(&config.crypto_hash_type, &mut crypto_cfg.crypto_hash_type)?; unsafe { // NOTE param order is 'wrong-way round' from C copy_nonoverlapping(config.private_key.as_ptr(), crypto_cfg.private_key.as_mut_ptr(), config.private_key.len()); } crypto_cfg.private_key_len = config.private_key.len() as u32; let res = unsafe { ffi::knet_handle_crypto_set_config(handle.knet_handle as ffi::knet_handle_t, &mut crypto_cfg, config_num) }; if res == 0 { Ok(()) } else { if res == -2 { Err(Error::other("Other cryto error")) } else { Err(Error::last_os_error()) } } } /// Whether to allow or disallow clear-text traffic when crypto is enabled with [handle_crypto_rx_clear_traffic] pub enum RxClearTraffic { Allow = 0, Disallow = 1, } /// Enable or disable clear-text traffic when crypto is enabled pub fn handle_crypto_rx_clear_traffic(handle: &Handle, value: RxClearTraffic) -> Result<()> { let c_value : u8 = match value { RxClearTraffic::Allow => 0, RxClearTraffic::Disallow => 1 }; let res = unsafe { ffi::knet_handle_crypto_rx_clear_traffic(handle.knet_handle as ffi::knet_handle_t, c_value) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Tell knet which crypto settings to use pub fn handle_crypto_use_config(handle: &Handle, config_num: u8) -> Result<()> { let res = unsafe { ffi::knet_handle_crypto_use_config(handle.knet_handle as ffi::knet_handle_t, config_num) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Set up packet compression pub fn handle_compress(handle: &Handle, config: &CompressConfig) -> Result<()> { let mut compress_cfg = ffi::knet_handle_compress_cfg { compress_model: [0; 16], compress_threshold : config.compress_threshold, compress_level : config.compress_level }; if config.compress_model.len() > 16 { return Err(Error::other("key too long")); } crate::string_to_bytes(&config.compress_model, &mut compress_cfg.compress_model)?; let res = unsafe { ffi::knet_handle_compress(handle.knet_handle as ffi::knet_handle_t, &mut compress_cfg) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Stats for the knet handle pub type HandleStats = ffi::knet_handle_stats; impl HandleStats { pub fn new() -> HandleStats { HandleStats { size: 0, tx_uncompressed_packets: 0, tx_compressed_packets: 0, tx_compressed_original_bytes: 0, tx_compressed_size_bytes: 0, tx_compress_time_ave: 0, tx_compress_time_min: 0, tx_compress_time_max: 0, tx_failed_to_compress: 0, tx_unable_to_compress: 0, rx_compressed_packets: 0, rx_compressed_original_bytes: 0, rx_compressed_size_bytes: 0, rx_compress_time_ave: 0, rx_compress_time_min: 0, rx_compress_time_max: 0, rx_failed_to_decompress: 0, tx_crypt_packets: 0, tx_crypt_byte_overhead: 0, tx_crypt_time_ave: 0, tx_crypt_time_min: 0, tx_crypt_time_max: 0, rx_crypt_packets: 0, rx_crypt_time_ave: 0, rx_crypt_time_min: 0, rx_crypt_time_max: 0, } } } impl Default for ffi::knet_handle_stats { fn default() -> Self { ffi::knet_handle_stats::new() } } impl fmt::Display for HandleStats { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}, ", self.tx_uncompressed_packets)?; write!(f, "{}, ", self.tx_compressed_packets)?; write!(f, "{}, ", self.tx_compressed_original_bytes)?; write!(f, "{}, ", self.tx_compressed_size_bytes)?; write!(f, "{}, ", self.tx_compress_time_ave)?; write!(f, "{}, ", self.tx_compress_time_min)?; write!(f, "{}, ", self.tx_compress_time_max)?; write!(f, "{}, ", self.tx_failed_to_compress)?; write!(f, "{}, ", self.tx_unable_to_compress)?; write!(f, "{}, ", self.rx_compressed_packets)?; write!(f, "{}, ", self.rx_compressed_original_bytes)?; write!(f, "{}, ", self.rx_compressed_size_bytes)?; write!(f, "{}, ", self.rx_compress_time_ave)?; write!(f, "{}, ", self.rx_compress_time_min)?; write!(f, "{}, ", self.rx_compress_time_max)?; write!(f, "{}, ", self.rx_failed_to_decompress)?; write!(f, "{}, ", self.tx_crypt_packets)?; write!(f, "{}, ", self.tx_crypt_byte_overhead)?; write!(f, "{}, ", self.tx_crypt_time_ave)?; write!(f, "{}, ", self.tx_crypt_time_min)?; write!(f, "{}, ", self.tx_crypt_time_max)?; write!(f, "{}, ", self.rx_crypt_packets)?; write!(f, "{}, ", self.rx_crypt_time_ave)?; write!(f, "{}, ", self.rx_crypt_time_min)?; write!(f, "{}, ", self.rx_crypt_time_max)?; Ok(()) } } /// Return statistics for this knet handle pub fn handle_get_stats(handle: &Handle) -> Result { let (res, stats) = unsafe { let mut c_stats = HandleStats::new(); let res = ffi::knet_handle_get_stats(handle.knet_handle as ffi::knet_handle_t, &mut c_stats, size_of::()); (res, c_stats) }; if res == 0 { Ok(stats) } else { Err(Error::last_os_error()) } } /// Tell [handle_clear_stats] whether to cleat all stats or just handle stats pub enum ClearStats { Handle = 1, HandleAndLink = 2, } /// Clear statistics pub fn handle_clear_stats(handle: &Handle, clear_options: ClearStats) -> Result<()> { let c_value : i32 = match clear_options { ClearStats::Handle => 1, ClearStats::HandleAndLink => 2 }; let res = unsafe { ffi::knet_handle_clear_stats(handle.knet_handle as ffi::knet_handle_t, c_value) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Crypto info returned from [get_crypto_list] pub struct CryptoInfo { pub name: String, pub properties: u8, // Unused } impl CryptoInfo { pub fn new(c_info: ffi::knet_crypto_info) -> CryptoInfo { let cstr = unsafe {CStr::from_ptr(c_info.name) }; let name = match cstr.to_str() { Ok(s) => s.to_string(), Err(e) => e.to_string(), }; CryptoInfo {properties: 0, name} } } /// Get a list of valid crypto options pub fn get_crypto_list() -> Result> { let mut list_entries: usize = 256; let mut c_list : [ffi::knet_crypto_info; 256] = [ ffi::knet_crypto_info{name: null(), properties: 0u8, pad:[0; 256]}; 256]; let res = unsafe { ffi::knet_get_crypto_list(&mut c_list[0], &mut list_entries) }; if res == 0 { let mut retvec = Vec::::new(); for i in c_list.iter().take(list_entries) { retvec.push(CryptoInfo::new(*i)); } Ok(retvec) } else { Err(Error::last_os_error()) } } /// Compressions types returned from [get_compress_list] pub struct CompressInfo { pub name: String, pub properties: u8, // Unused } impl CompressInfo { pub fn new(c_info: ffi::knet_compress_info) -> CompressInfo { let cstr = unsafe {CStr::from_ptr(c_info.name) }; let name = match cstr.to_str() { Ok(s) => s.to_string(), Err(e) => e.to_string(), }; CompressInfo {properties: 0, name} } } /// Return a list of compression options pub fn get_compress_list() -> Result> { let mut list_entries: usize = 256; let mut c_list : [ffi::knet_compress_info; 256] = [ ffi::knet_compress_info{name: null(), properties: 0u8, pad:[0; 256]}; 256]; let res = unsafe { ffi::knet_get_compress_list(&mut c_list[0], &mut list_entries) }; if res == 0 { let mut retvec = Vec::::new(); for i in c_list.iter().take(list_entries) { retvec.push(CompressInfo::new(*i)); } Ok(retvec) } else { Err(Error::last_os_error()) } } /// Enable callback when the onwire version for a node changes pub fn handle_enable_onwire_ver_notify(handle: &Handle, private_data: u64, onwire_notify_fn: Option) -> Result<()> { // This looks a bit different to the other _enable*_notify calls because knet // calls the callback function in the API. Which results in a deadlock with our // own mutex if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(handle.knet_handle)) { h.onwire_notify_private_data = private_data; h.onwire_notify_fn = onwire_notify_fn; } else { return Err(Error::other("Rust handle not found")); }; let res = match onwire_notify_fn { Some(_f) => unsafe { ffi::knet_handle_enable_onwire_ver_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, Some(rust_onwire_notify_fn)) }, None => unsafe { ffi::knet_handle_enable_onwire_ver_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, None) }, }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the onsure version for a node pub fn handle_get_onwire_ver(handle: &Handle, host_id: &HostId) -> Result<(u8,u8,u8)> { let mut onwire_min_ver = 0u8; let mut onwire_max_ver = 0u8; let mut onwire_ver = 0u8; let res = unsafe { ffi::knet_handle_get_onwire_ver(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, &mut onwire_min_ver, &mut onwire_max_ver, &mut onwire_ver) }; if res == 0 { Ok((onwire_min_ver, onwire_max_ver, onwire_ver)) } else { Err(Error::last_os_error()) } } /// Set the onsire version for this node pub fn handle_set_onwire_ver(handle: &Handle, onwire_ver: u8) -> Result<()> { let res = unsafe { ffi::knet_handle_set_onwire_ver(handle.knet_handle as ffi::knet_handle_t, onwire_ver) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Set the reconnect interval. pub fn handle_set_transport_reconnect_interval(handle: &Handle, msecs: u32) -> Result<()> { let res = unsafe { ffi::knet_handle_set_transport_reconnect_interval(handle.knet_handle as ffi::knet_handle_t, msecs) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the reconnect interval. pub fn handle_get_transport_reconnect_interval(handle: &Handle) -> Result { let mut msecs = 0u32; let res = unsafe { ffi::knet_handle_get_transport_reconnect_interval(handle.knet_handle as ffi::knet_handle_t, &mut msecs) }; if res == 0 { Ok(msecs) } else { Err(Error::last_os_error()) } } /// Add a new host ID pub fn host_add(handle: &Handle, host_id: &HostId) -> Result<()> { let res = unsafe { ffi::knet_host_add(handle.knet_handle as ffi::knet_handle_t, host_id.host_id) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Remove a Host ID pub fn host_remove(handle: &Handle, host_id: &HostId) -> Result<()> { let res = unsafe { ffi::knet_host_remove(handle.knet_handle as ffi::knet_handle_t, host_id.host_id) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Set the name of a host pub fn host_set_name(handle: &Handle, host_id: &HostId, name: &str) -> Result<()> { let c_name = CString::new(name)?; let res = unsafe { ffi::knet_host_set_name(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, c_name.as_ptr()) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } const KNET_MAX_HOST_LEN:usize = 256; const KNET_MAX_PORT_LEN:usize = 6; /// Retrieve the name of a host given its ID pub fn host_get_name_by_host_id(handle: &Handle, host_id: &HostId) -> Result { let mut c_name: [c_char; KNET_MAX_HOST_LEN] = [0; KNET_MAX_HOST_LEN]; let res = unsafe { ffi::knet_host_get_name_by_host_id(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, c_name.as_mut_ptr()) }; if res == 0 { crate::string_from_bytes(c_name.as_ptr(), KNET_MAX_HOST_LEN) } else { Err(Error::last_os_error()) } } /// Return the ID of a host given its name pub fn host_get_id_by_host_name(handle: &Handle, name: &str) -> Result { let c_name = CString::new(name)?; let mut c_host_id = 0u16; let res = unsafe { ffi::knet_host_get_id_by_host_name(handle.knet_handle as ffi::knet_handle_t, c_name.as_ptr(), &mut c_host_id) }; if res == 0 { Ok(HostId{host_id: c_host_id}) } else { Err(Error::last_os_error()) } } const KNET_MAX_HOST: usize = 65536; /// Return a list of host IDs known to this handle pub fn host_get_host_list(handle: &Handle) -> Result> { let mut c_host_ids: [u16; KNET_MAX_HOST] = [0; KNET_MAX_HOST]; let mut c_host_ids_entries: usize = 0; let res = unsafe { ffi::knet_host_get_host_list(handle.knet_handle as ffi::knet_handle_t, &mut c_host_ids[0], &mut c_host_ids_entries) }; if res == 0 { let mut host_vec = Vec::::new(); for i in c_host_ids.iter().take(c_host_ids_entries) { host_vec.push(HostId {host_id: *i}); } Ok(host_vec) } else { Err(Error::last_os_error()) } } /// Link Policies for [host_set_policy] #[derive(Copy, Clone, PartialEq, Eq)] pub enum LinkPolicy { Passive, Active, Rr, } impl LinkPolicy{ pub fn new(value: u8) -> LinkPolicy { match value { 2 => LinkPolicy::Rr, 1 => LinkPolicy::Active, _ => LinkPolicy::Passive, } } pub fn to_u8(self: LinkPolicy) -> u8 { match self { LinkPolicy::Passive => 0, LinkPolicy::Active => 1, LinkPolicy::Rr => 2, } } } impl fmt::Display for LinkPolicy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { LinkPolicy::Passive => write!(f, "Passive"), LinkPolicy::Active => write!(f, "Active"), LinkPolicy::Rr => write!(f, "RR"), } } } /// Set the policy for this host, this only makes sense if multiple links between hosts are configured pub fn host_set_policy(handle: &Handle, host_id: &HostId, policy: LinkPolicy) -> Result<()> { let c_value: u8 = policy.to_u8(); let res = unsafe { ffi::knet_host_set_policy(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, c_value) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Return the current link policy for a node pub fn host_get_policy(handle: &Handle, host_id: &HostId) -> Result { let mut c_value: u8 = 0; let res = unsafe { ffi::knet_host_get_policy(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, &mut c_value) }; if res == 0 { Ok(LinkPolicy::new(c_value)) } else { Err(Error::last_os_error()) } } /// Current status of a host. remote & reachable are current not used pub struct HostStatus { pub reachable: bool, pub remote: bool, pub external: bool, } impl fmt::Display for HostStatus { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "reachable: {}, ", self.reachable)?; write!(f, "remote: {}, ", self.remote)?; write!(f, "external: {}", self.external)?; Ok(()) } } /// Return the current status of a host pub fn host_get_status(handle: &Handle, host_id: &HostId) -> Result { let mut c_value = ffi::knet_host_status { reachable:0, remote:0, external:0}; let res = unsafe { ffi::knet_host_get_status(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, &mut c_value) }; if res == 0 { Ok(HostStatus { reachable: crate::u8_to_bool(c_value.reachable), remote: crate::u8_to_bool(c_value.remote), external: crate::u8_to_bool(c_value.external) }) } else { Err(Error::last_os_error()) } } /// Enable callbacks when the status of a host changes pub fn host_enable_status_change_notify(handle: &Handle, private_data: u64, host_status_change_notify_fn: Option) -> Result<()> { if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(handle.knet_handle)) { h.host_status_change_notify_private_data = private_data; h.host_status_change_notify_fn = host_status_change_notify_fn; let res = match host_status_change_notify_fn { Some(_f) => unsafe { ffi::knet_host_enable_status_change_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, Some(rust_host_status_change_notify_fn)) }, None => unsafe { ffi::knet_host_enable_status_change_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, None) }, }; if res == 0 { return Ok(()); } else { return Err(Error::last_os_error()); } } Err(Error::other("Rust handle not found")) } /// Transport types supported in knet pub enum TransportId { Loopback, Udp, - Sctp, } impl TransportId { pub fn new(id: u8) -> TransportId { match id { - 2 => TransportId::Sctp, 1 => TransportId::Udp, _ => TransportId::Loopback, } } pub fn to_u8(self: &TransportId) -> u8 { match self { TransportId::Loopback => 0, TransportId::Udp => 1, - TransportId::Sctp => 2, } } pub fn to_string(self: &TransportId) -> String { match self { TransportId::Udp => "UDP".to_string(), - TransportId::Sctp => "SCTP".to_string(), TransportId::Loopback => "Loopback".to_string() } } pub fn from_string(name: String) -> TransportId { match name.as_str() { "UDP" => TransportId::Udp, - "SCTP" => TransportId::Sctp, "Loopback" => TransportId::Loopback, _ => TransportId::Loopback, } } } /// Transport info returned from [get_transport_list] pub struct TransportInfo { pub name: String, pub id: TransportId, pub properties: u8, // currently unused } // Controversially implementing name_by_id and id_by_name here impl TransportInfo { pub fn new(c_info: ffi::knet_transport_info) -> TransportInfo { let cstr = unsafe {CStr::from_ptr(c_info.name) }; let name = match cstr.to_str() { Ok(s) => s.to_string(), Err(e) => e.to_string(), }; TransportInfo {properties: 0, id: TransportId::new(c_info.id), name} } } pub fn get_transport_list() -> Result> { let mut list_entries: usize = 256; let mut c_list : [ffi::knet_transport_info; 256] = [ ffi::knet_transport_info{name: null(), id: 0u8, properties: 0u8, pad:[0; 256]}; 256]; let res = unsafe { ffi::knet_get_transport_list(&mut c_list[0], &mut list_entries) }; if res == 0 { let mut retvec = Vec::::new(); for i in c_list.iter().take(list_entries) { retvec.push(TransportInfo::new(*i)); } Ok(retvec) } else { Err(Error::last_os_error()) } } /// Configure a link to a host ID. dst_addr may be None for a dynamic link. pub fn link_set_config(handle: &Handle, host_id: &HostId, link_id: u8, transport: TransportId, src_addr: &SocketAddr, dst_addr: Option<&SocketAddr>, flags: LinkFlags) -> Result<()> { // Not really mut, but C is dumb let mut c_srcaddr = make_new_sockaddr_storage(src_addr); // dst_addr can be NULL/None if this is a dynamic link let res = if let Some(dst) = dst_addr { let mut c_dstaddr = make_new_sockaddr_storage(dst); unsafe { ffi::knet_link_set_config(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, transport.to_u8(), &mut c_srcaddr, &mut c_dstaddr, flags.bits()) } } else { unsafe { ffi::knet_link_set_config(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, transport.to_u8(), &mut c_srcaddr, null_mut(), flags.bits()) } }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Return a link's configuration pub fn link_get_config(handle: &Handle, host_id: &HostId, link_id: u8) -> Result<(TransportId, Option, Option, LinkFlags)> { let mut c_srcaddr = OsSocketAddr::new(); let mut c_dstaddr = OsSocketAddr::new(); let mut c_transport = 0u8; let mut c_flags = 0u64; let mut c_dynamic = 0u8; let res = unsafe { ffi::knet_link_get_config(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_transport, c_srcaddr.as_mut_ptr() as *mut ffi::sockaddr_storage, c_dstaddr.as_mut_ptr() as *mut ffi::sockaddr_storage, &mut c_dynamic, &mut c_flags) }; if res == 0 { let r_transport = TransportId::new(c_transport); Ok((r_transport, c_srcaddr.into(), c_dstaddr.into(), LinkFlags::from_bits(c_flags).unwrap_or(LinkFlags::empty()))) } else { Err(Error::last_os_error()) } } /// Clear a link configuration. pub fn link_clear_config(handle: &Handle, host_id: &HostId, link_id: u8) -> Result<()> { let res = unsafe { ffi::knet_link_clear_config(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Type of ACL pub enum AclAcceptReject { Accept, Reject, } impl AclAcceptReject { pub fn new(ar: u32) -> AclAcceptReject { match ar { ffi::CHECK_ACCEPT => AclAcceptReject::Accept, ffi::CHECK_REJECT => AclAcceptReject::Reject, _ => AclAcceptReject::Reject, } } pub fn to_u32(self: &AclAcceptReject) -> u32 { match self { AclAcceptReject::Accept => ffi::CHECK_ACCEPT, AclAcceptReject::Reject => ffi::CHECK_REJECT, } } } /// What the ACL should check pub enum AclCheckType { Address, Mask, Range, } impl AclCheckType { pub fn new(ct: u32) -> AclCheckType { match ct { ffi::CHECK_TYPE_ADDRESS => AclCheckType::Address, ffi::CHECK_TYPE_MASK => AclCheckType::Mask, ffi::CHECK_TYPE_RANGE => AclCheckType::Range, _ => AclCheckType::Address, } } pub fn to_u32(self: &AclCheckType) -> u32 { match self { AclCheckType::Address => ffi::CHECK_TYPE_ADDRESS, AclCheckType::Mask => ffi::CHECK_TYPE_MASK, AclCheckType::Range => ffi::CHECK_TYPE_RANGE, } } } // We need to have a zeroed-out stackaddr storage to pass to the ACL APIs // as knet compares the whole sockaddr_storage when using knet_rm_acl() fn make_new_sockaddr_storage(ss: &SocketAddr) -> ffi::sockaddr_storage { // A blank one #[cfg(target_os="freebsd")] let mut new_ss = ffi::sockaddr_storage { ss_family: 0, ss_len: 0, __ss_pad1: [0; 6], __ss_align: 0, __ss_pad2: [0; 112], }; #[cfg(target_os="linux")] let mut new_ss = ffi::sockaddr_storage { ss_family: 0, __ss_padding: [0; 118], __ss_align: 0, }; #[cfg(target_os="illumos")] let mut new_ss = ffi::sockaddr_storage { ss_family: 0, _ss_pad1: [0; 6], _ss_align: 0.0, _ss_pad2: [0; 240], }; let p_new_ss : *mut ffi::sockaddr_storage = &mut new_ss; // Rust only fills in what it thinks is necessary let c_ss : OsSocketAddr = (*ss).into(); // Copy it unsafe { // Only copy as much as is in the OsSocketAddr copy_nonoverlapping(c_ss.as_ptr(), p_new_ss as *mut libc::sockaddr, 1); } new_ss } /// Add an ACL to a link, adds the ACL to the end of the list. pub fn link_add_acl(handle: &Handle, host_id: &HostId, link_id: u8, ss1: &SocketAddr, ss2: &SocketAddr, check_type: AclCheckType, acceptreject: AclAcceptReject) -> Result<()> { // Not really mut, but C is dumb let mut c_ss1 = make_new_sockaddr_storage(ss1); let mut c_ss2 = make_new_sockaddr_storage(ss2); let res = unsafe { ffi::knet_link_add_acl(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_ss1, &mut c_ss2, check_type.to_u32(), acceptreject.to_u32()) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Insert an ACL anywhere in the ACL list for this host/link pub fn link_insert_acl(handle: &Handle, host_id: &HostId, link_id: u8, index: i32, ss1: &SocketAddr, ss2: &SocketAddr, check_type: AclCheckType, acceptreject: AclAcceptReject) -> Result<()> { // Not really mut, but C is dumb let mut c_ss1 = make_new_sockaddr_storage(ss1); let mut c_ss2 = make_new_sockaddr_storage(ss2); let res = unsafe { ffi::knet_link_insert_acl(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, index, &mut c_ss1, &mut c_ss2, check_type.to_u32(), acceptreject.to_u32()) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Remove an ACL for this host/link pub fn link_rm_acl(handle: &Handle, host_id: &HostId, link_id: u8, ss1: &SocketAddr, ss2: &SocketAddr, check_type: AclCheckType, acceptreject: AclAcceptReject) -> Result<()> { // Not really mut, but C is dumb let mut c_ss1 = make_new_sockaddr_storage(ss1); let mut c_ss2 = make_new_sockaddr_storage(ss2); let res = unsafe { ffi::knet_link_rm_acl(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_ss1, &mut c_ss2, check_type.to_u32(), acceptreject.to_u32()) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Clear out all ACLs from this host/link pub fn link_clear_acl(handle: &Handle, host_id: &HostId, link_id: u8) -> Result<()> { let res = unsafe { ffi::knet_link_clear_acl(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Enable/disable a link (you still need to call [handle_setfwd] for traffic to flow pub fn link_set_enable(handle: &Handle, host_id: &HostId, link_id: u8, enable: bool) -> Result<()> { let res = unsafe { ffi::knet_link_set_enable(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, enable as u32) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the 'enabled' status for a link pub fn link_get_enable(handle: &Handle, host_id: &HostId, link_id: u8) -> Result { let mut c_enable = 0u32; let res = unsafe { ffi::knet_link_get_enable(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_enable) }; if res == 0 { Ok(crate::u32_to_bool(c_enable)) } else { Err(Error::last_os_error()) } } /// Set the ping timers for a link pub fn link_set_ping_timers(handle: &Handle, host_id: &HostId, link_id: u8, interval: i64, timeout: i64, precision: u32) -> Result<()> { let res = unsafe { ffi::knet_link_set_ping_timers(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, interval, timeout, precision) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the ping timers for a link pub fn link_get_ping_timers(handle: &Handle, host_id: &HostId, link_id: u8) -> Result<(i64, i64, u32)> { let mut c_interval : ffi::time_t = 0; let mut c_timeout : ffi::time_t = 0; let mut c_precision = 0u32; let res = unsafe { ffi::knet_link_get_ping_timers(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_interval, &mut c_timeout, &mut c_precision) }; if res == 0 { Ok((c_interval, c_timeout, c_precision)) } else { Err(Error::last_os_error()) } } /// Set the pong count for a link pub fn link_set_pong_count(handle: &Handle, host_id: &HostId, link_id: u8, pong_count: u8) -> Result<()> { let res = unsafe { ffi::knet_link_set_pong_count(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, pong_count) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the pong count for a link pub fn link_get_pong_count(handle: &Handle, host_id: &HostId, link_id: u8) -> Result { let mut c_pong_count = 0u8; let res = unsafe { ffi::knet_link_get_pong_count(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_pong_count) }; if res == 0 { Ok(c_pong_count) } else { Err(Error::last_os_error()) } } /// Set the link priority (only useful with multiple links to a node) pub fn link_set_priority(handle: &Handle, host_id: &HostId, link_id: u8, priority: u8) -> Result<()> { let res = unsafe { ffi::knet_link_set_priority(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, priority) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the link priority pub fn link_get_priority(handle: &Handle, host_id: &HostId, link_id: u8) -> Result { let mut c_priority = 0u8; let res = unsafe { ffi::knet_link_get_priority(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_priority) }; if res == 0 { Ok(c_priority) } else { Err(Error::last_os_error()) } } const KNET_MAX_LINK: usize = 8; /// Get a list of links for this host pub fn link_get_link_list(handle: &Handle, host_id: &HostId) -> Result> { let mut c_link_ids: [u8; KNET_MAX_LINK] = [0; KNET_MAX_LINK]; let mut c_link_ids_entries: usize = 0; let res = unsafe { ffi::knet_link_get_link_list(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, &mut c_link_ids[0], &mut c_link_ids_entries) }; if res == 0 { let mut link_vec = Vec::::new(); for i in c_link_ids.iter().take(c_link_ids_entries) { link_vec.push(*i); } Ok(link_vec) } else { Err(Error::last_os_error()) } } /// Enable callbacks when a link status changes pub fn link_enable_status_change_notify(handle: &Handle, private_data: u64, link_status_change_notify_fn: Option) -> Result<()> { if let Some(h) = HANDLE_HASH.lock().unwrap().get_mut(&(handle.knet_handle)) { h.link_status_change_notify_private_data = private_data; h.link_status_change_notify_fn = link_status_change_notify_fn; let res = match link_status_change_notify_fn { Some(_f) => unsafe { ffi::knet_link_enable_status_change_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, Some(rust_link_status_change_notify_fn)) }, None => unsafe { ffi::knet_link_enable_status_change_notify(handle.knet_handle as ffi::knet_handle_t, handle.knet_handle as *mut c_void, None) }, }; if res == 0 { return Ok(()); } else { return Err(Error::last_os_error()); } } Err(Error::other("Rust handle not found")) } /// Link stats pub struct LinkStats { pub tx_data_packets: u64, pub rx_data_packets: u64, pub tx_data_bytes: u64, pub rx_data_bytes: u64, pub rx_ping_packets: u64, pub tx_ping_packets: u64, pub rx_ping_bytes: u64, pub tx_ping_bytes: u64, pub rx_pong_packets: u64, pub tx_pong_packets: u64, pub rx_pong_bytes: u64, pub tx_pong_bytes: u64, pub rx_pmtu_packets: u64, pub tx_pmtu_packets: u64, pub rx_pmtu_bytes: u64, pub tx_pmtu_bytes: u64, pub tx_total_packets: u64, pub rx_total_packets: u64, pub tx_total_bytes: u64, pub rx_total_bytes: u64, pub tx_total_errors: u64, pub tx_total_retries: u64, pub tx_pmtu_errors: u32, pub tx_pmtu_retries: u32, pub tx_ping_errors: u32, pub tx_ping_retries: u32, pub tx_pong_errors: u32, pub tx_pong_retries: u32, pub tx_data_errors: u32, pub tx_data_retries: u32, pub latency_min: u32, pub latency_max: u32, pub latency_ave: u32, pub latency_samples: u32, pub down_count: u32, pub up_count: u32, pub last_up_times: Vec, pub last_down_times: Vec, } // Quick & Dirty printing impl fmt::Display for LinkStats { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}, ", self.tx_data_packets)?; write!(f, "{}, ", self.rx_data_packets)?; write!(f, "{}, ", self.tx_data_bytes)?; write!(f, "{}, ", self.rx_data_bytes)?; write!(f, "{}, ", self.rx_ping_packets)?; write!(f, "{}, ", self.tx_ping_packets)?; write!(f, "{}, ", self.rx_ping_bytes)?; write!(f, "{}, ", self.tx_ping_bytes)?; write!(f, "{}, ", self.rx_pong_packets)?; write!(f, "{}, ", self.tx_pong_packets)?; write!(f, "{}, ", self.rx_pong_bytes)?; write!(f, "{}, ", self.tx_pong_bytes)?; write!(f, "{}, ", self.rx_pmtu_packets)?; write!(f, "{}, ", self.tx_pmtu_packets)?; write!(f, "{}, ", self.rx_pmtu_bytes)?; write!(f, "{}, ", self.tx_pmtu_bytes)?; write!(f, "{}, ", self.tx_total_packets)?; write!(f, "{}, ", self.rx_total_packets)?; write!(f, "{}, ", self.tx_total_bytes)?; write!(f, "{}, ", self.rx_total_bytes)?; write!(f, "{}, ", self.tx_total_errors)?; write!(f, "{}, ", self.tx_total_retries)?; write!(f, "{}, ", self.tx_pmtu_errors)?; write!(f, "{}, ", self.tx_pmtu_retries)?; write!(f, "{}, ", self.tx_ping_errors)?; write!(f, "{}, ", self.tx_ping_retries)?; write!(f, "{}, ", self.tx_pong_errors)?; write!(f, "{}, ", self.tx_pong_retries)?; write!(f, "{}, ", self.tx_data_errors)?; write!(f, "{}, ", self.tx_data_retries)?; write!(f, "{}, ", self.latency_min)?; write!(f, "{}, ", self.latency_max)?; write!(f, "{}, ", self.latency_ave)?; write!(f, "{}, ", self.latency_samples)?; write!(f, "{}, ", self.down_count)?; write!(f, "{}, ", self.up_count)?; write!(f, "Last up times: ")?; // There's no sensible print for SystemTime in the std library // and I don't want to add dependancies here for printing as it // mostly going to be the client's responsibility, so use the Debug option for i in &self.last_up_times { write!(f, "{i:?}")?; } write!(f, " Last down times: ")?; for i in &self.last_down_times { write!(f, "{i:?}")?; } Ok(()) } } // I wish this all wasn't necessary! impl ffi::knet_link_stats { pub fn new() -> ffi::knet_link_stats { ffi::knet_link_stats { tx_data_packets: 0, rx_data_packets: 0, tx_data_bytes: 0, rx_data_bytes: 0, rx_ping_packets: 0, tx_ping_packets: 0, rx_ping_bytes: 0, tx_ping_bytes: 0, rx_pong_packets: 0, tx_pong_packets: 0, rx_pong_bytes: 0, tx_pong_bytes: 0, rx_pmtu_packets: 0, tx_pmtu_packets: 0, rx_pmtu_bytes: 0, tx_pmtu_bytes: 0, tx_total_packets: 0, rx_total_packets: 0, tx_total_bytes: 0, rx_total_bytes: 0, tx_total_errors: 0, tx_total_retries: 0, tx_pmtu_errors: 0, tx_pmtu_retries: 0, tx_ping_errors: 0, tx_ping_retries: 0, tx_pong_errors: 0, tx_pong_retries: 0, tx_data_errors: 0, tx_data_retries: 0, latency_min: 0, latency_max: 0, latency_ave: 0, latency_samples: 0, down_count: 0, up_count: 0, last_up_times: [0; 16], last_down_times: [0; 16], last_up_time_index: 0, last_down_time_index: 0, } } } impl Default for ffi::knet_link_stats { fn default() -> Self { ffi::knet_link_stats::new() } } impl ffi::knet_link_status { pub fn new()-> ffi::knet_link_status { ffi::knet_link_status { size: 0, src_ipaddr : [0; KNET_MAX_HOST_LEN], dst_ipaddr : [0; KNET_MAX_HOST_LEN], src_port : [0; KNET_MAX_PORT_LEN], dst_port : [0; KNET_MAX_PORT_LEN], enabled: 0, connected: 0, dynconnected: 0, pong_last: ffi::timespec{ tv_sec: 0, tv_nsec: 0}, mtu: 0, proto_overhead: 0, stats: ffi::knet_link_stats::new(), } } } impl Default for ffi::knet_link_status { fn default() -> Self { ffi::knet_link_status::new() } } /// Link status (includes a [LinkStats]) pub struct LinkStatus { pub src_ipaddr: String, pub dst_ipaddr: String, pub src_port: String, pub dst_port: String, pub enabled: bool, pub connected: bool, pub dynconnected: bool, pub pong_last: SystemTime, pub mtu: u32, pub proto_overhead: u32, pub stats: LinkStats, } impl LinkStatus { pub fn new(c_stats: ffi::knet_link_status) -> LinkStatus { LinkStatus { src_ipaddr : crate::string_from_bytes_safe(c_stats.src_ipaddr.as_ptr(), KNET_MAX_HOST_LEN), src_port : crate::string_from_bytes_safe(c_stats.src_port.as_ptr(), KNET_MAX_HOST_LEN), dst_ipaddr : crate::string_from_bytes_safe(c_stats.dst_ipaddr.as_ptr(), KNET_MAX_HOST_LEN), dst_port : crate::string_from_bytes_safe(c_stats.dst_port.as_ptr(), KNET_MAX_HOST_LEN), enabled : crate::u8_to_bool(c_stats.enabled), connected : crate::u8_to_bool(c_stats.connected), dynconnected : crate::u8_to_bool(c_stats.dynconnected), pong_last : systemtime_from_timespec(c_stats.pong_last), mtu : c_stats.mtu, proto_overhead : c_stats.proto_overhead, stats : LinkStats::new(c_stats.stats), } } } impl fmt::Display for LinkStatus { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "src_ip_addr: {}:{}, ", self.src_ipaddr, self.src_port)?; write!(f, "dst_ip_addr: {}:{}, ", self.dst_ipaddr, self.src_port)?; write!(f, "enabled: {}, connected: {}, mtu: {}, overhead: {}, ", self.enabled, self.connected, self.mtu, self.proto_overhead)?; write!(f, "stats: {}", self.stats)?; Ok(()) } } fn systemtime_from_time_t(t: u64) -> SystemTime { SystemTime::UNIX_EPOCH+Duration::from_secs(t) } fn systemtime_from_timespec(t: ffi::timespec) -> SystemTime { SystemTime::UNIX_EPOCH+Duration::from_secs(t.tv_sec as u64) +Duration::from_nanos(t.tv_nsec as u64) // TODO may panic?? } fn copy_circular_buffer_of_link_events(num: usize, times: &[ffi::time_t]) -> Vec { let mut times_vec = Vec::::new(); for index in (0 .. num).rev() { if times[index] == 0 { break } times_vec.push(systemtime_from_time_t(times[index] as u64)); // TODO may panic?? } for index in (num+1 .. MAX_LINK_EVENTS).rev() { if times[index] == 0 { break; } times_vec.push(systemtime_from_time_t(times[index] as u64)); // TODO may panic?? } times_vec } const MAX_LINK_EVENTS: usize = 16; impl LinkStats { pub fn new(cstats: ffi::knet_link_stats) -> LinkStats { let up_times = copy_circular_buffer_of_link_events(cstats.last_up_time_index as usize, &cstats.last_up_times); let down_times = copy_circular_buffer_of_link_events(cstats.last_down_time_index as usize, &cstats.last_down_times); LinkStats { tx_data_packets: cstats.tx_data_packets, rx_data_packets: cstats.rx_data_packets, tx_data_bytes: cstats.tx_data_bytes, rx_data_bytes: cstats.rx_data_bytes, rx_ping_packets: cstats.rx_ping_packets, tx_ping_packets: cstats.tx_ping_packets, rx_ping_bytes: cstats.rx_ping_bytes, tx_ping_bytes: cstats.tx_ping_bytes, rx_pong_packets: cstats.rx_pong_packets, tx_pong_packets: cstats.tx_pong_packets, rx_pong_bytes: cstats.rx_pong_bytes, tx_pong_bytes: cstats.tx_pong_bytes, rx_pmtu_packets: cstats.rx_pmtu_packets, tx_pmtu_packets: cstats.tx_pmtu_packets, rx_pmtu_bytes: cstats.rx_pmtu_bytes, tx_pmtu_bytes: cstats.tx_pmtu_bytes, tx_total_packets: cstats.tx_total_packets, rx_total_packets: cstats.rx_total_packets, tx_total_bytes: cstats.tx_total_bytes, rx_total_bytes: cstats.rx_total_bytes, tx_total_errors: cstats.tx_total_errors, tx_total_retries: cstats.tx_total_retries, tx_pmtu_errors: cstats.tx_pmtu_errors, tx_pmtu_retries: cstats.tx_pmtu_retries, tx_ping_errors: cstats.tx_ping_errors, tx_ping_retries: cstats.tx_ping_retries, tx_pong_errors: cstats.tx_pong_errors, tx_pong_retries: cstats.tx_pong_retries, tx_data_errors: cstats.tx_data_errors, tx_data_retries: cstats.tx_data_retries, latency_min: cstats.latency_min, latency_max: cstats.latency_max, latency_ave: cstats.latency_ave, latency_samples: cstats.latency_samples, down_count: cstats.down_count, up_count: cstats.up_count, last_up_times: up_times, last_down_times: down_times, } } } /// Get the status (and stats) of a link pub fn link_get_status(handle: &Handle, host_id: &HostId, link_id: u8) -> Result { let (res, stats) = unsafe { let mut c_stats : ffi::knet_link_status = ffi::knet_link_status::new(); let res = ffi::knet_link_get_status(handle.knet_handle as ffi::knet_handle_t, host_id.host_id, link_id, &mut c_stats, size_of::()); (res, c_stats) }; if res == 0 { let r_status = LinkStatus::new(stats); Ok(r_status) } else { Err(Error::last_os_error()) } } /// Get the logging subsystem ID given its name pub fn log_get_subsystem_id(name: &str) -> Result { let c_name = CString::new(name)?; let res = unsafe { ffi::knet_log_get_subsystem_id(c_name.as_ptr()) }; Ok(res) } /// Get the logging subsystem name given its ID pub fn log_get_subsystem_name(id: u8) -> Result { let res = unsafe { ffi::knet_log_get_subsystem_name(id) }; crate::string_from_bytes(res, 256) } /// Get the name of a logging level pub fn log_get_loglevel_id(name: &str) -> Result { let c_name = CString::new(name)?; let res = unsafe { ffi::knet_log_get_loglevel_id(c_name.as_ptr()) }; Ok(res) } /// Get the ID of a logging level, given its name pub fn log_get_loglevel_name(id: u8) -> Result { let res = unsafe { ffi::knet_log_get_loglevel_name(id) }; crate::string_from_bytes(res, 256) } /// Logging levels pub enum LogLevel { Err, Warn, Info, Debug, Trace, } impl LogLevel { pub fn new(level: u8) -> LogLevel { match level { 0 => LogLevel::Err, 1 => LogLevel::Warn, 2 => LogLevel::Info, 3 => LogLevel::Debug, _ => LogLevel::Trace // 4=Trace, but default anything to it too } } pub fn to_u8(self: &LogLevel) -> u8 { match self { LogLevel::Err => 0, LogLevel::Warn => 1, LogLevel::Info => 2, LogLevel::Debug => 3, LogLevel::Trace => 4, } } } impl fmt::Display for LogLevel { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { LogLevel::Err => write!(f, "Err"), LogLevel::Warn => write!(f, "Warn"), LogLevel::Info => write!(f, "Info"), LogLevel::Debug => write!(f, "Debug"), LogLevel::Trace => write!(f, "Trace"), } } } /// Subsystems known to the knet logger pub enum SubSystem { Common, Handle, Host, Listener, Link, Transport, Crypto, Compress, Filter, Dstcache, Heartbeat, Pmtud, Tx, Rx, TranspBase, TranspLoopback, TranspUdp, - TranspSctp, NssCrypto, OpensslCrypto, Zlibcomp, Lz4comp, Lz4hccomp, Lzo2comp, Lzmacomp, Bzip2comp, Zstdcomp, Unknown, } impl SubSystem { pub fn to_u8(self: &SubSystem) -> u8 { match self { SubSystem::Common => ffi::KNET_SUB_COMMON, SubSystem::Handle => ffi::KNET_SUB_HANDLE, SubSystem::Host => ffi::KNET_SUB_HOST, SubSystem::Listener => ffi::KNET_SUB_LISTENER, SubSystem::Link => ffi::KNET_SUB_LINK, SubSystem::Transport => ffi::KNET_SUB_TRANSPORT, SubSystem::Crypto => ffi::KNET_SUB_CRYPTO, SubSystem::Compress => ffi::KNET_SUB_COMPRESS, SubSystem::Filter => ffi::KNET_SUB_FILTER, SubSystem::Dstcache => ffi::KNET_SUB_DSTCACHE, SubSystem::Heartbeat => ffi::KNET_SUB_HEARTBEAT, SubSystem::Pmtud => ffi::KNET_SUB_PMTUD, SubSystem::Tx => ffi::KNET_SUB_TX, SubSystem::Rx => ffi::KNET_SUB_RX, SubSystem::TranspBase => ffi::KNET_SUB_TRANSP_BASE, SubSystem::TranspLoopback => ffi::KNET_SUB_TRANSP_LOOPBACK, SubSystem::TranspUdp => ffi::KNET_SUB_TRANSP_UDP, - SubSystem::TranspSctp => ffi::KNET_SUB_TRANSP_SCTP, SubSystem::NssCrypto => ffi::KNET_SUB_NSSCRYPTO, SubSystem::OpensslCrypto => ffi::KNET_SUB_OPENSSLCRYPTO, SubSystem::Zlibcomp => ffi::KNET_SUB_ZLIBCOMP, SubSystem::Lz4comp => ffi::KNET_SUB_LZ4COMP, SubSystem::Lz4hccomp => ffi::KNET_SUB_LZ4HCCOMP, SubSystem::Lzo2comp => ffi::KNET_SUB_LZO2COMP, SubSystem::Lzmacomp => ffi::KNET_SUB_LZMACOMP, SubSystem::Bzip2comp => ffi::KNET_SUB_BZIP2COMP, SubSystem::Zstdcomp => ffi::KNET_SUB_ZSTDCOMP, SubSystem::Unknown => ffi::KNET_SUB_UNKNOWN, } } pub fn new(subsys: u8) -> SubSystem { match subsys { 1 => SubSystem::Unknown, 2 => SubSystem::Unknown, _ => SubSystem::Unknown, } } } /// Set the current logging level pub fn log_set_loglevel(handle: &Handle, subsystem: SubSystem, level: LogLevel) -> Result<()> { let c_level = level.to_u8(); let c_subsys = subsystem.to_u8(); let res = unsafe { ffi::knet_log_set_loglevel(handle.knet_handle as ffi::knet_handle_t, c_subsys, c_level) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } /// Get the current logging level pub fn log_get_loglevel(handle: &Handle, subsystem: SubSystem) -> Result { let mut c_level:u8 = 0; let c_subsys = subsystem.to_u8(); let res = unsafe { ffi::knet_log_get_loglevel(handle.knet_handle as ffi::knet_handle_t, c_subsys, &mut c_level) }; if res == 0 { Ok(LogLevel::new(c_level)) } else { Err(Error::last_os_error()) } } /// Use dscp for IP_TOS on socket to implement KNET_LINK_FLAG_TRAFFICHIPRIO pub fn handle_setprio_dscp(handle: &Handle, dscp: u8) -> Result<()> { let res = unsafe { ffi::knet_handle_setprio_dscp(handle.knet_handle as ffi::knet_handle_t, dscp) }; if res == 0 { Ok(()) } else { Err(Error::last_os_error()) } } diff --git a/libknet/internals.h b/libknet/internals.h index 084f2a02..d3cb7f08 100644 --- a/libknet/internals.h +++ b/libknet/internals.h @@ -1,498 +1,498 @@ /* * Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under LGPL-2.0+ */ #ifndef __KNET_INTERNALS_H__ #define __KNET_INTERNALS_H__ /* * NOTE: you shouldn't need to include this header normally */ #include #include #include #include "libknet.h" #include "onwire.h" #include "compat.h" #include "threads_common.h" #define KNET_DATABUFSIZE KNET_MAX_PACKET_SIZE + KNET_HEADER_ALL_SIZE #define KNET_DATABUFSIZE_CRYPT_PAD 1024 #define KNET_DATABUFSIZE_CRYPT KNET_DATABUFSIZE + KNET_DATABUFSIZE_CRYPT_PAD #define KNET_DATABUFSIZE_COMPRESS_PAD 1024 #define KNET_DATABUFSIZE_COMPRESS KNET_DATABUFSIZE + KNET_DATABUFSIZE_COMPRESS_PAD #define KNET_RING_RCVBUFF 8388608 #define PCKT_FRAG_MAX UINT8_MAX #define PCKT_RX_BUFS 512 #define KNET_EPOLL_MAX_EVENTS KNET_DATAFD_MAX + 1 /* * Size of threads stack. Value is choosen by experimenting, how much is needed * to sucesfully finish test suite, and at the time of writing patch it was * ~300KiB. To have some room for future enhancement it is increased * by factor of 3 and rounded. */ #define KNET_THREAD_STACK_SIZE (1024 * 1024) typedef void *knet_transport_link_t; /* per link transport handle */ typedef void *knet_transport_t; /* per knet_h transport handle */ struct knet_transport_ops; /* Forward because of circular dependancy */ struct knet_mmsghdr { struct msghdr msg_hdr; /* Message header */ unsigned int msg_len; /* Number of bytes transmitted */ }; struct knet_link { /* required */ struct sockaddr_storage src_addr; struct sockaddr_storage dst_addr; /* configurable */ unsigned int dynamic; /* see KNET_LINK_DYN_ define above */ uint8_t priority; /* higher priority == preferred for A/P */ unsigned long long ping_interval; /* interval */ unsigned long long pong_timeout; /* timeout */ unsigned long long pong_timeout_adj; /* timeout adjusted for latency */ uint8_t pong_timeout_backoff; /* see link.h for definition */ unsigned int latency_max_samples; /* precision */ uint8_t pong_count; /* how many ping/pong to send/receive before link is up */ uint64_t flags; void *access_list_match_entry_head; /* pointer to access list match_entry list head */ /* status */ struct knet_link_status status; /* internals */ pthread_mutex_t link_stats_mutex; /* used to update link stats */ uint8_t link_id; uint8_t transport; /* #defined constant from API */ knet_transport_link_t transport_link; /* link_info_t from transport */ int outsock; unsigned int configured:1; /* set to 1 if src/dst have been configured transport initialized on this link*/ unsigned int transport_connected:1; /* set to 1 if lower level transport is connected */ uint8_t received_pong; struct timespec ping_last; /* used by PMTUD thread as temp per-link variables and should always contain the onwire_len value! */ - uint32_t proto_overhead; /* IP + UDP/SCTP overhead. NOT to be confused + uint32_t proto_overhead; /* IP + UDP overhead. NOT to be confused with stats.proto_overhead that includes also knet headers and crypto headers */ struct timespec pmtud_last; uint32_t last_ping_size; uint32_t last_good_mtu; uint32_t last_bad_mtu; uint32_t last_sent_mtu; uint32_t last_recv_mtu; uint32_t pmtud_crypto_timeout_multiplier;/* used by PMTUd to adjust timeouts on high loads */ uint8_t has_valid_mtu; }; #define KNET_CBUFFER_SIZE 4096 struct knet_host_defrag_buf { char buf[KNET_DATABUFSIZE]; uint8_t in_use; /* 0 buffer is free, 1 is in use */ seq_num_t pckt_seq; /* identify the pckt we are receiving */ uint8_t frag_recv; /* how many frags did we receive */ uint8_t frag_map[PCKT_FRAG_MAX];/* bitmap of what we received? */ uint8_t last_first; /* special case if we receive the last fragment first */ ssize_t frag_size; /* normal frag size (not the last one) */ ssize_t last_frag_size; /* the last fragment might not be aligned with MTU size */ struct timespec last_update; /* keep time of the last pckt */ }; struct knet_host { /* required */ knet_node_id_t host_id; /* configurable */ uint8_t link_handler_policy; char name[KNET_MAX_HOST_LEN]; /* status */ struct knet_host_status status; /* * onwire info */ uint8_t onwire_ver; /* node current onwire version */ uint8_t onwire_max_ver; /* node supports up to this version */ /* internals */ char circular_buffer[KNET_CBUFFER_SIZE]; seq_num_t rx_seq_num; seq_num_t untimed_rx_seq_num; seq_num_t timed_rx_seq_num; uint8_t got_data; /* defrag/reassembly buffers */ struct knet_host_defrag_buf *defrag_bufs; uint16_t allocated_defrag_bufs; /* track use % of allocated defrag buffers */ uint8_t in_use_defrag_buffers[UINT8_MAX]; uint8_t in_use_defrag_buffers_samples; uint8_t in_use_defrag_buffers_index; char circular_buffer_defrag[KNET_CBUFFER_SIZE]; /* link stuff */ struct knet_link link[KNET_MAX_LINK]; uint8_t active_link_entries; uint8_t active_links[KNET_MAX_LINK]; struct knet_host *next; }; struct knet_sock { int sockfd[2]; /* sockfd[0] will always be application facing * and sockfd[1] internal if sockpair has been created by knet */ int is_socket; /* check if it's a socket for recvmmsg usage */ int is_created; /* knet created this socket and has to clean up on exit/del */ int in_use; /* set to 1 if it's use, 0 if free */ int has_error; /* set to 1 if there were errors reading from the sock * and socket has been removed from epoll */ int flags; /* KNET_DATAFD_FLAGs */ }; struct knet_fd_trackers { - uint8_t transport; /* transport type (UDP/SCTP...) */ + uint8_t transport; /* transport type (UDP...) */ uint8_t data_type; /* internal use for transport to define what data are associated * with this fd */ socklen_t sockaddr_len; /* Size of sockaddr_in[6] structure for this socket */ void *data; /* pointer to the data */ int ifindex; /* interface index for this bound address */ }; #define KNET_MAX_FDS KNET_MAX_HOST * KNET_MAX_LINK * 4 #define KNET_MAX_COMPRESS_METHODS UINT8_MAX #define KNET_MAX_CRYPTO_INSTANCES 2 struct knet_handle_stats_extra { uint64_t tx_crypt_pmtu_packets; uint64_t tx_crypt_pmtu_reply_packets; uint64_t tx_crypt_ping_packets; uint64_t tx_crypt_pong_packets; }; #define KNET_RX_ODD_PACKETS_THRESHOLD 20 #define KNET_USAGE_SAMPLES_DEFAULT UINT8_MAX #define KNET_USAGE_SAMPLES_TIMESPAN_DEFAULT 10 /* seconds */ struct knet_handle { knet_node_id_t host_id; unsigned int enabled:1; struct knet_sock sockfd[KNET_DATAFD_MAX + 1]; int logfd; uint8_t log_levels[KNET_MAX_SUBSYSTEMS]; int dstsockfd[2]; int send_to_links_epollfd; int recv_from_links_epollfd; int dst_link_handler_epollfd; uint8_t use_access_lists; /* set to 0 for disable, 1 for enable */ unsigned int pmtud_interval; unsigned int manual_mtu; unsigned int data_mtu; /* contains the max data size that we can send onwire * without frags */ uint8_t prio_dscp; /* use this dscp value for KNET_LINK_FLAG_TRAFFICHIPRIO */ struct knet_host *host_head; struct knet_host *host_index[KNET_MAX_HOST]; knet_transport_t transports[KNET_MAX_TRANSPORTS+1]; struct knet_fd_trackers knet_transport_fd_tracker[KNET_MAX_FDS]; /* track status for each fd handled by transports */ struct knet_handle_stats stats; struct knet_handle_stats_extra stats_extra; pthread_mutex_t handle_stats_mutex; /* used to protect handle stats */ uint32_t reconnect_int; knet_node_id_t host_ids[KNET_MAX_HOST]; size_t host_ids_entries; struct knet_header *recv_from_sock_buf; struct knet_header *send_to_links_buf[PCKT_FRAG_MAX]; struct knet_header *recv_from_links_buf[PCKT_RX_BUFS]; struct knet_header *pingbuf; struct knet_header *pmtudbuf; uint8_t threads_status[KNET_THREAD_MAX]; uint8_t threads_flush_queue[KNET_THREAD_MAX]; useconds_t threads_timer_res; pthread_mutex_t threads_status_mutex; pthread_t send_to_links_thread; pthread_t recv_from_links_thread; pthread_t heartbt_thread; pthread_t dst_link_handler_thread; pthread_t pmtud_link_handler_thread; pthread_rwlock_t global_rwlock; /* global config lock */ pthread_mutex_t pmtud_mutex; /* pmtud mutex to handle conditional send/recv + timeout */ pthread_cond_t pmtud_cond; /* conditional for above */ pthread_mutex_t tx_mutex; /* used to protect knet_send_sync and TX thread */ pthread_mutex_t hb_mutex; /* used to protect heartbeat thread and seq_num broadcasting */ pthread_mutex_t backoff_mutex; /* used to protect dst_link->pong_timeout_adj */ pthread_mutex_t kmtu_mutex; /* used to protect kernel_mtu */ pthread_mutex_t onwire_mutex; /* used to protect onwire version */ uint8_t onwire_ver; /* currently agreed onwire version across known nodes */ uint8_t onwire_min_ver; /* min and max are constant and don´t need any mutex protection. */ uint8_t onwire_max_ver; /* we define them as part of internal handle so that we can mingle with them for testing purposes */ uint8_t onwire_force_ver; /* manually configure onwire_ver */ uint8_t onwire_ver_remap; /* when this is on, all mapping will use version 1 for now */ uint8_t rx_odd_packets; /* used to warn if too many weird packets are being received */ uint32_t kernel_mtu; /* contains the MTU detected by the kernel on a given link */ int pmtud_waiting; int pmtud_running; int pmtud_forcerun; int pmtud_abort; struct crypto_instance *crypto_instance[KNET_MAX_CRYPTO_INSTANCES + 1]; /* store an extra pointer to allow 0|1|2 values without too much magic in the code */ uint8_t crypto_in_use_config; /* crypto config to use for TX */ uint8_t crypto_only; /* allow only crypto (1) or also clear (0) traffic */ size_t sec_block_size; size_t sec_hash_size; size_t sec_salt_size; unsigned char *send_to_links_buf_crypt[PCKT_FRAG_MAX]; unsigned char *recv_from_links_buf_crypt; unsigned char *recv_from_links_buf_decrypt; unsigned char *pingbuf_crypt; unsigned char *pmtudbuf_crypt; int compress_model; int compress_level; size_t compress_threshold; void *compress_int_data[KNET_MAX_COMPRESS_METHODS]; /* for compress method private data */ unsigned char *recv_from_links_buf_decompress; unsigned char *send_to_links_buf_compress; seq_num_t tx_seq_num; pthread_mutex_t tx_seq_num_mutex; uint16_t defrag_bufs_min; uint16_t defrag_bufs_max; uint8_t defrag_bufs_shrink_threshold; uint8_t defrag_bufs_usage_samples; uint8_t defrag_bufs_usage_samples_timespan; defrag_bufs_reclaim_policy_t defrag_bufs_reclaim_policy; struct timespec defrag_bufs_last_run; uint8_t has_loop_link; uint8_t loop_link; void *dst_host_filter_fn_private_data; int (*dst_host_filter_fn) ( void *private_data, const unsigned char *outdata, ssize_t outdata_len, uint8_t tx_rx, knet_node_id_t this_host_id, knet_node_id_t src_node_id, int8_t *channel, knet_node_id_t *dst_host_ids, size_t *dst_host_ids_entries); void *pmtud_notify_fn_private_data; void (*pmtud_notify_fn) ( void *private_data, unsigned int data_mtu); void *host_status_change_notify_fn_private_data; void (*host_status_change_notify_fn) ( void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external); void *link_status_change_notify_fn_private_data; void (*link_status_change_notify_fn) ( void *private_data, knet_node_id_t host_id, uint8_t link_id, uint8_t connected, uint8_t remote, uint8_t external); void *sock_notify_fn_private_data; void (*sock_notify_fn) ( void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno); void *onwire_ver_notify_fn_private_data; void (*onwire_ver_notify_fn) ( void *private_data, uint8_t onwire_min_ver, uint8_t onwire_max_ver, uint8_t onwire_ver); int fini_in_progress; uint64_t flags; struct qb_list_head list; const char *plugin_path; }; struct handle_tracker { struct qb_list_head head; }; /* * lib_config stuff shared across everything */ extern pthread_rwlock_t shlib_rwlock; /* global shared lib load lock */ extern pthread_mutex_t handle_config_mutex; extern struct handle_tracker handle_list; extern uint8_t handle_list_init; int _is_valid_handle(knet_handle_t knet_h); int _init_shlib_tracker(knet_handle_t knet_h); void _fini_shlib_tracker(void); /* * NOTE: every single operation must be implementend * for every protocol. */ /* - * for now knet supports only IP protocols (udp/sctp) + * for now knet supports only IP protocols (udp) * in future there might be others like ARP * or TIPC. * keep this around as transport information * to use for access lists and other operations */ #define TRANSPORT_PROTO_LOOPBACK 0 #define TRANSPORT_PROTO_IP_PROTO 1 /* - * some transports like SCTP can filter incoming + * some transports (none that are supported though!) can filter incoming * connections before knet has to process * any packets. * GENERIC_ACL -> packet has to be read and filterted * PROTO_ACL -> transport provides filtering at lower levels * and packet does not need to be processed */ typedef enum { USE_NO_ACL, USE_GENERIC_ACL, USE_PROTO_ACL } transport_acl; /* * make it easier to map values in transports.c */ #define TRANSPORT_PROTO_NOT_CONNECTION_ORIENTED 0 #define TRANSPORT_PROTO_IS_CONNECTION_ORIENTED 1 /* * Returns from transport_tx_sock_error */ typedef enum { KNET_TRANSPORT_RX_ERROR = -1, KNET_TRANSPORT_SOCK_ERROR_INTERNAL = -1, KNET_TRANSPORT_SOCK_ERROR_IGNORE = 0, KNET_TRANSPORT_SOCK_ERROR_RETRY = 1, } transport_sock_error_t; /* * Returns from transport_rx_is_data */ typedef enum { KNET_TRANSPORT_RX_ISDATA_ERROR = -1, KNET_TRANSPORT_RX_NOT_DATA_CONTINUE = 0, KNET_TRANSPORT_RX_NOT_DATA_STOP = 1, KNET_TRANSPORT_RX_IS_DATA = 2, - /* These two are only really used by SCTP */ + /* These two were only really used by SCTP ? */ KNET_TRANSPORT_RX_OOB_DATA_CONTINUE = 3, KNET_TRANSPORT_RX_OOB_DATA_STOP = 4 } transport_rx_isdata_t; /* * All functions that return int return 0 for success, and -1 for failure. */ typedef struct knet_transport_ops { /* * transport generic information */ const char *transport_name; const uint8_t transport_id; const uint8_t built_in; uint8_t transport_protocol; transport_acl transport_acl_type; /* * connection oriented protocols like SCTP * don´t need dst_addr in sendto calls and * on some OSes are considered EINVAL. */ uint8_t transport_is_connection_oriented; uint32_t transport_mtu_overhead; /* * transport init must allocate the new transport * and perform all internal initializations * (threads, lists, etc). */ int (*transport_init)(knet_handle_t knet_h); /* * transport free must releases _all_ resources * allocated by tranport_init */ int (*transport_free)(knet_handle_t knet_h); /* * link operations should take care of all the * sockets and epoll management for a given link/transport set * transport_link_disable should return -1 and errno = EBUSY * if listener is still in use, and any other errno in case * the link cannot be disabled. * * set_config/clear_config are invoked in global write lock context */ int (*transport_link_set_config)(knet_handle_t knet_h, struct knet_link *link); int (*transport_link_clear_config)(knet_handle_t knet_h, struct knet_link *link); /* * transport callback for incoming dynamic connections * this is called in global read lock context */ int (*transport_link_dyn_connect)(knet_handle_t knet_h, int sockfd, struct knet_link *link); /* * per transport error handling of recvmmsg * (see _handle_recv_from_links comments for details) */ /* * transport_rx_sock_error is invoked when recvmmsg returns <= 0 * * transport_rx_sock_error is invoked with both global_rdlock */ int (*transport_rx_sock_error)(knet_handle_t knet_h, int sockfd, int recv_err, int recv_errno); /* * transport_tx_sock_error is invoked with global_rwlock and * it's invoked when sendto or sendmmsg returns =< 0 * * it should return a transport_sock_error_t * any sleep or wait action should happen inside the transport code */ transport_sock_error_t (*transport_tx_sock_error)(knet_handle_t knet_h, int sockfd, int subsys, int recv_err, int recv_errno); /* * this function is called on _every_ received packet * to verify if the packet is data or internal protocol error handling * * it should return a transport_tx_isdata_t * * transport_rx_is_data is invoked with both global_rwlock * and fd_tracker read lock (from RX thread) */ transport_rx_isdata_t (*transport_rx_is_data)(knet_handle_t knet_h, int sockfd, struct knet_mmsghdr *msg); /* * this function is called by links.c when a link down event is recorded * to notify the transport that packets are not going through, and give * transport the opportunity to take actions. */ int (*transport_link_is_down)(knet_handle_t knet_h, struct knet_link *link); } knet_transport_ops_t; struct pretty_names { const char *name; uint8_t val; }; #endif diff --git a/libknet/libknet.h b/libknet/libknet.h index 575efc3c..122f7056 100644 --- a/libknet/libknet.h +++ b/libknet/libknet.h @@ -1,2743 +1,2741 @@ /* * Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under LGPL-2.0+ */ #ifndef __LIBKNET_H__ #define __LIBKNET_H__ #include #include #include #include #include #include /** * @file libknet.h * @brief kronosnet API include file * @copyright Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. * * Kronosnet is an advanced VPN system for High Availability applications. */ #define KNET_API_VER 2 /* * libknet limits */ /** typedef for a knet node */ typedef uint16_t knet_node_id_t; /* * Maximum number of hosts */ #define KNET_MAX_HOST 65536 /* * Maximum number of links between 2 hosts */ #define KNET_MAX_LINK 8 /* * Maximum packet size that should be written to datafd * see knet_handle_new for details */ #define KNET_MAX_PACKET_SIZE 65536 /* * Buffers used for pretty logging * host is used to store both ip addresses and hostnames */ #define KNET_MAX_HOST_LEN 256 #define KNET_MAX_PORT_LEN 6 /* * Some notifications can be generated either on TX or RX */ #define KNET_NOTIFY_TX 0 #define KNET_NOTIFY_RX 1 /* * Link flags */ /* * Where possible, set traffic priority to high. * On Linux this sets the TOS to INTERACTIVE (6), * see tc-prio(8) for more infomation * A dscp value may be configured, see knet_handle_setprio_dscp. */ #define KNET_LINK_FLAG_TRAFFICHIPRIO (1ULL << 0) /* * Handle flags */ /* * Use privileged operations during socket setup. */ #define KNET_HANDLE_FLAG_PRIVILEGED (1ULL << 0) /* * Flags that affect what appears (and should be provided) on the datafd */ #define KNET_DATAFD_FLAG_RX_RETURN_INFO (1ULL << 0) /* * threads timer resolution (see knet_handle_set_threads_timer_res below) */ #define KNET_THREADS_TIMER_RES 200000 /** * Opaque handle for this knet connection, created with knet_handle_new() and * freed with knet_handle_free() */ typedef struct knet_handle *knet_handle_t; /* * Handle structs/API calls */ /** * knet_handle_new * * @brief create a new instance of a knet handle * * host_id - Each host in a knet is identified with a unique * ID. when creating a new handle local host_id * must be specified (0 to UINT16_MAX are all valid). * It is the user's responsibility to check that the value * is unique, or bad things might happen. * * log_fd - Write file descriptor. If set to a value > 0, it will be used * to write log packets from libknet to the application. * Setting to 0 will disable logging from libknet. * It is possible to enable logging at any given time (see logging API). * Make sure to either read from this filedescriptor properly and/or * mark it O_NONBLOCK, otherwise if the fd becomes full, libknet could * block. * It is strongly encouraged to use pipes (ex: pipe(2) or pipe2(2)) for * logging fds due to the atomic nature of writes between fds. * See also libknet test suite for reference and guidance. * The caller is responsible for management of the FD. eg. knet will not * close it when knet_handle_free(3) is called * * default_log_level - * If logfd is specified, it will initialize all subsystems to log * at default_log_level value. (see logging API) * * flags - bitwise OR of some of the following flags: * KNET_HANDLE_FLAG_PRIVILEGED: use privileged operations setting up the * communication sockets. If disabled, failure to acquire large * enough socket buffers is ignored but logged. Inadequate buffers * lead to poor performance. * * @return * on success, a new knet_handle_t is returned. * on failure, NULL is returned and errno is set. * knet-specific errno values: * ENAMETOOLONG - socket buffers couldn't be set big enough and KNET_HANDLE_FLAG_PRIVILEGED was specified * ERANGE - buffer size readback returned unexpected type */ knet_handle_t knet_handle_new(knet_node_id_t host_id, int log_fd, uint8_t default_log_level, uint64_t flags); /** * knet_handle_free * * @brief Destroy a knet handle, free all resources * * knet_h - pointer to knet_handle_t * * @return * knet_handle_free returns * 0 on success * -1 on error and errno is set. */ int knet_handle_free(knet_handle_t knet_h); /** * knet_handle_set_threads_timer_res * * @brief Change internal thread timer resolution * * knet_h - pointer to knet_handle_t * * timeres - some threads inside knet will use usleep(timeres) * to check if any activity has to be performed, or wait * for the next cycle. 'timeres' (expressed in nano seconds) * defines this interval, with a default of KNET_THREADS_TIMER_RES * (200000). * The lower this value is, the more often knet will perform * those checks and allows a more (time) precise execution of * some operations (for example ping/pong), at the cost of higher * CPU usage. * Accepted values: * 0 - reset timer res to default * 1 - 999 invalid (as it would cause 100% CPU spinning on some * epoll operations) * 1000 or higher - valid * * Unless you know exactly what you are doing, stay away from * changing the default or seek written and notarized approval * from the knet developer team. * * @return * knet_handle_set_threads_timer_res returns * 0 on success * -1 on error and errno is set. */ int knet_handle_set_threads_timer_res(knet_handle_t knet_h, useconds_t timeres); /** * knet_handle_get_threads_timer_res * * @brief Get internal thread timer resolutions * * knet_h - pointer to knet_handle_t * * timeres - current timer res value * * @return * knet_handle_set_threads_timer_res returns * 0 on success and timerres will contain the current value * -1 on error and errno is set. */ int knet_handle_get_threads_timer_res(knet_handle_t knet_h, useconds_t *timeres); /** * knet_handle_enable_sock_notify * * @brief Register a callback to receive socket events * * knet_h - pointer to knet_handle_t * * sock_notify_fn_private_data * void pointer to data that can be used to identify * the callback. * * sock_notify_fn * A callback function that is invoked every time * a socket in the datafd pool will report an error (-1) * or an end of read (0) (see socket.7). * This function MUST NEVER block or add substantial delays. * The callback is invoked in an internal unlocked area * to allow calls to knet_handle_add_datafd/knet_handle_remove_datafd * to swap/replace the bad fd. * if both err and errno are 0, it means that the socket * has received a 0 byte packet (EOF?). * The callback function must either remove the fd from knet * (by calling knet_handle_remove_fd()) or dup a new fd in its place. * Failure to do this can cause problems. * * @return * knet_handle_enable_sock_notify returns * 0 on success * -1 on error and errno is set. */ int knet_handle_enable_sock_notify(knet_handle_t knet_h, void *sock_notify_fn_private_data, void (*sock_notify_fn) ( void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)); /* sorry! can't call it errno ;) */ #define KNET_DATAFD_MAX 32 struct knet_datafd_header { /** Size of the structure. Used for backwards compatibilty. Only use fields up to the size you were compiled with, but advance to the end of the struct using the size field here */ size_t size; /** nodeid of node sending this message */ knet_node_id_t src_nodeid; }; /** * knet_handle_add_datafd * * @brief Install a file descriptor for communication * * IMPORTANT: In order to add datafd to knet, knet_handle_enable_sock_notify * _MUST_ be set and be able to handle both errors (-1) and * 0 bytes read / write from the provided datafd. * On read error (< 0) from datafd, the socket is automatically * removed from polling to avoid spinning on dead sockets. * It is safe to call knet_handle_remove_datafd even on sockets * that have been removed. * * knet_h - pointer to knet_handle_t * * *datafd - read/write file descriptor. * knet will read data here to send to the other hosts * and will write data received from the network. * Each data packet can be of max size KNET_MAX_PACKET_SIZE! * Applications using knet_send/knet_recv will receive a * proper error if the packet size is not within boundaries. * Applications using their own functions to write to the * datafd should NOT write more than KNET_MAX_PACKET_SIZE. * * Please refer to handle.c on how to set up a socketpair. * * datafd can be 0, and knet_handle_add_datafd will create a properly * populated socket pair the same way as ping_test, or a value * higher than 0. A negative number will return an error. * On exit knet_handle_free will take care to cleanup the * socketpair only if they have been created by knet_handle_add_datafd. * * It is possible to pass either sockets or normal fds. * User provided datafd will be marked as non-blocking and close-on-exec. * * *channel - This value is analogous to the tag in VLAN tagging. * A negative value will auto-allocate a channel. * Setting a value between 0 and 31 will try to allocate that * specific channel (unless already in use). * * It is possible to add up to 32 datafds but be aware that each * one of them must have a receiving end on the other host. * * Example: * hostA channel 0 will be delivered to datafd on hostB channel 0 * hostA channel 1 to hostB channel 1. * * Each channel must have a unique file descriptor. * * If your application could have 2 channels on one host and one * channel on another host, then you can use dst_host_filter * to manipulate channel values on TX and RX. * flags - Bitwise OR of any of the following: * - KNET_ADD_DAFATA_FLAG_RX_RETURN_INFO * * @return * knet_handle_add_datafd returns * @retval 0 on success, * *datafd will be populated with a socket if the original value was 0 * or if a specific fd was set, the value is untouched. * *channel will be populated with a channel number if the original value * was negative or the value is untouched if a specific channel * was requested. * * @retval -1 on error and errno is set. * *datafd and *channel are untouched or empty. */ int knet_handle_add_datafd(knet_handle_t knet_h, int *datafd, int8_t *channel, uint32_t flags); /** * knet_handle_remove_datafd * * @brief Remove a file descriptor from knet * * knet_h - pointer to knet_handle_t * * datafd - file descriptor to remove. * NOTE that if the socket/fd was created by knet_handle_add_datafd, * the socket will be closed by libknet. * * @return * knet_handle_remove_datafd returns * 0 on success * -1 on error and errno is set. */ int knet_handle_remove_datafd(knet_handle_t knet_h, int datafd); /** * knet_handle_get_channel * * @brief Get the channel associated with a file descriptor * * knet_h - pointer to knet_handle_t * * datafd - get the channel associated to this datafd * * *channel - will contain the result * * @return * knet_handle_get_channel returns * @retval 0 on success * and *channel will contain the result * @retval -1 on error and errno is set. * and *channel content is meaningless */ int knet_handle_get_channel(knet_handle_t knet_h, const int datafd, int8_t *channel); /** * knet_handle_get_datafd * * @brief Get the file descriptor associated with a channel * * knet_h - pointer to knet_handle_t * * channel - get the datafd associated to this channel * * *datafd - will contain the result * * @return * knet_handle_get_datafd returns * @retval 0 on success * and *datafd will contain the results * @retval -1 on error and errno is set. * and *datafd content is meaningless */ int knet_handle_get_datafd(knet_handle_t knet_h, const int8_t channel, int *datafd); /** * knet_recv * * @brief Receive data from knet nodes * * knet_h - pointer to knet_handle_t * * buff - pointer to buffer to store the received data * * buff_len - buffer length * * channel - channel number * * @return * knet_recv is a commodity function to wrap iovec operations * around a socket. It returns a call to readv(2). */ ssize_t knet_recv(knet_handle_t knet_h, char *buff, const size_t buff_len, const int8_t channel); /** * knet_send * * @brief Send data to knet nodes * * knet_h - pointer to knet_handle_t * * buff - pointer to the buffer of data to send * * buff_len - length of data to send * * channel - channel number * * @return * knet_send is a commodity function to wrap iovec operations * around a socket. It returns a call to writev(2). */ ssize_t knet_send(knet_handle_t knet_h, const char *buff, const size_t buff_len, const int8_t channel); /** * knet_send_sync * * @brief Synchronously send data to knet nodes * * knet_h - pointer to knet_handle_t * * buff - pointer to the buffer of data to send * * buff_len - length of data to send * * channel - data channel to use (see knet_handle_add_datafd(3)) * * All knet RX/TX operations are async for performance reasons. * There are applications that might need a sync version of data * transmission and receive errors in case of failure to deliver * to another host. * knet_send_sync bypasses the whole TX async layer and delivers * data directly to the link layer, and returns errors accordingly. * knet_send_sync sends only one packet to one host at a time. * It does NOT support multiple destinations or multicast packets. * Decision is still based on dst_host_filter_fn. * * @return * knet_send_sync returns 0 on success and -1 on error. * In addition to normal sendmmsg errors, knet_send_sync can fail * due to: * * @retval ECANCELED - data forward is disabled * @retval EFAULT - dst_host_filter fatal error * @retval EINVAL - dst_host_filter did not provide dst_host_ids_entries on unicast pckts * @retval E2BIG - dst_host_filter did return more than one dst_host_ids_entries on unicast pckts * @retval ENOMSG - received unknown message type * @retval EHOSTDOWN - unicast pckt cannot be delivered because dest host is not connected yet * @retval ECHILD - crypto failed * @retval EAGAIN - sendmmsg was unable to send all messages and there was no progress during retry * @retval ENETDOWN - a packet filter was not installed (necessary for knet_send_sync, but not knet_send) */ int knet_send_sync(knet_handle_t knet_h, const char *buff, const size_t buff_len, const int8_t channel); /** * knet_handle_enable_filter * * @brief install a filter to route packets * * knet_h - pointer to knet_handle_t * * dst_host_filter_fn_private_data * void pointer to data that can be used to identify * the callback. * * dst_host_filter_fn - * is a callback function that is invoked every time * a packet hits datafd (see knet_handle_new(3)). * the function allows users to tell libknet where the * packet has to be delivered. * * const unsigned char *outdata - is a pointer to the * current packet * ssize_t outdata_len - length of the above data * uint8_t tx_rx - filter is called on tx or rx * (KNET_NOTIFY_TX, KNET_NOTIFY_RX) * knet_node_id_t this_host_id - host_id processing the packet * knet_node_id_t src_host_id - host_id that generated the * packet * knet_node_id_t *dst_host_ids - array of KNET_MAX_HOST knet_node_id_t * where to store the destinations * (uninitialized by caller, callee should never * read it) * size_t *dst_host_ids_entries - number of hosts to send the message * * dst_host_filter_fn should return * -1 on error, packet is discarded. * 0 packet is unicast and should be sent to dst_host_ids and there are * dst_host_ids_entries in the buffer. * 1 packet is broadcast/multicast and is sent all hosts. * contents of dst_host_ids and dst_host_ids_entries are ignored. * * @return * knet_handle_enable_filter returns * 0 on success * -1 on error and errno is set. */ int knet_handle_enable_filter(knet_handle_t knet_h, void *dst_host_filter_fn_private_data, int (*dst_host_filter_fn) ( void *private_data, const unsigned char *outdata, ssize_t outdata_len, uint8_t tx_rx, knet_node_id_t this_host_id, knet_node_id_t src_host_id, int8_t *channel, knet_node_id_t *dst_host_ids, size_t *dst_host_ids_entries)); /** * knet_handle_setfwd * * @brief Start packet forwarding * * knet_h - pointer to knet_handle_t * * enable - set to 1 to allow data forwarding, 0 to disable data forwarding. * * @return * knet_handle_setfwd returns * 0 on success * -1 on error and errno is set. * * By default data forwarding is off and no traffic will pass through knet until * it is set on. */ int knet_handle_setfwd(knet_handle_t knet_h, unsigned int enabled); /** * knet_handle_setprio_dscp * * @brief Use dscp for IP_TOS on socket to implement KNET_LINK_FLAG_TRAFFICHIPRIO * * knet_h - pointer to knet_handle_t * * dscp - dscp value to set on all new sockets * * This function must be called prior to configure knet links. * * It disables the use of IPTOS_LOWDELAY and uses the given dscp value in the * IP header's TOS field instead. * * Setting dscp to 0 reverts to using IPTOS_LOWDELAY. * * @return * knet_handle_setprio_dscp returns * 0 on success * -1 on error and errno is set. */ int knet_handle_setprio_dscp(knet_handle_t knet_h, uint8_t dscp); /** * knet_handle_enable_access_lists * * @brief Enable or disable usage of access lists (default: off) * * knet_h - pointer to knet_handle_t * * enable - set to 1 to use access lists, 0 to disable access_lists. * * @return * knet_handle_enable_access_lists returns * 0 on success * -1 on error and errno is set. * * access lists are bound to links. There are 2 types of links: * 1) point to point, where both source and destinations are well known * at configuration time. * 2) open links, where only the source is known at configuration time. * * knet will automatically generate access lists for point to point links. * * For open links, knet provides 4 API calls to manipulate access lists: * knet_link_add_acl(3), knet_link_rm_acl(3), knet_link_insert_acl(3) * and knet_link_clear_acl(3). * Those API calls will work exclusively on open links as they * are of no use on point to point links. * * knet will not enforce any access list unless specifically enabled by * knet_handle_enable_access_lists(3). * * From a security / programming perspective we recommend: * - create the knet handle * - enable access lists * - configure hosts and links * - configure access lists for open links */ int knet_handle_enable_access_lists(knet_handle_t knet_h, unsigned int enabled); #define KNET_PMTUD_DEFAULT_INTERVAL 60 /** * knet_handle_pmtud_setfreq * * @brief Set the interval between PMTUd scans * * knet_h - pointer to knet_handle_t * * interval - define the interval in seconds between PMTUd scans * range from 1 to 86400 (24h) * * @return * knet_handle_pmtud_setfreq returns * 0 on success * -1 on error and errno is set. * * default interval is 60. */ int knet_handle_pmtud_setfreq(knet_handle_t knet_h, unsigned int interval); /** * knet_handle_pmtud_getfreq * * @brief Get the interval between PMTUd scans * * knet_h - pointer to knet_handle_t * * interval - pointer where to store the current interval value * * @return * knet_handle_pmtud_setfreq returns * 0 on success * -1 on error and errno is set. */ int knet_handle_pmtud_getfreq(knet_handle_t knet_h, unsigned int *interval); /** * knet_handle_enable_pmtud_notify * * @brief install a callback to receive PMTUd changes * * knet_h - pointer to knet_handle_t * * pmtud_notify_fn_private_data * void pointer to data that can be used to identify * the callback. * * pmtud_notify_fn * is a callback function that is invoked every time * a path MTU size change is detected. * The function allows libknet to notify the user * of data MTU, that's the max value that can be send * onwire without fragmentation. The data MTU will always * be lower than real link MTU because it accounts for * protocol overhead, knet packet header and (if configured) * crypto overhead, * This function MUST NEVER block or add substantial delays. * * @return * knet_handle_enable_pmtud_notify returns * 0 on success * -1 on error and errno is set. */ int knet_handle_enable_pmtud_notify(knet_handle_t knet_h, void *pmtud_notify_fn_private_data, void (*pmtud_notify_fn) ( void *private_data, unsigned int data_mtu)); /** * knet_handle_pmtud_set * * @brief Set the current interface MTU * * knet_h - pointer to knet_handle_t * * iface_mtu - current interface MTU, value 0 to 65535. 0 will * re-enable automatic MTU discovery. * In a setup with multiple interfaces, please specify * the lowest MTU between the selected intefaces. * knet will automatically adjust this value for * all headers overhead and set the correct data_mtu. * data_mtu can be retrivied with knet_handle_pmtud_get(3) * or applications will receive a pmtud_notify event * if enabled via knet_handle_enable_pmtud_notify(3). * * @return * knet_handle_pmtud_set returns * 0 on success * -1 on error and errno is set. */ int knet_handle_pmtud_set(knet_handle_t knet_h, unsigned int iface_mtu); /** * knet_handle_pmtud_get * * @brief Get the current data MTU * * knet_h - pointer to knet_handle_t * * data_mtu - pointer where to store data_mtu * * @return * knet_handle_pmtud_get returns * 0 on success * -1 on error and errno is set. */ int knet_handle_pmtud_get(knet_handle_t knet_h, unsigned int *data_mtu); #define KNET_MIN_KEY_LEN 128 #define KNET_MAX_KEY_LEN 4096 /** * Structure passed into knet_handle_set_crypto_config() to determine * the crypto options to use for the current communications handle */ struct knet_handle_crypto_cfg { /** Model to use. nss, openssl, etc */ char crypto_model[16]; /** Cipher type name for encryption. aes 256 etc */ char crypto_cipher_type[16]; /** Hash type for digest. sha512 etc */ char crypto_hash_type[16]; /** Private key */ unsigned char private_key[KNET_MAX_KEY_LEN]; /** Length of private key */ unsigned int private_key_len; }; /** * knet_handle_crypto_set_config * * @brief set up packet cryptographic signing & encryption * * knet_h - pointer to knet_handle_t * * knet_handle_crypto_cfg - * pointer to a knet_handle_crypto_cfg structure * * crypto_model should contain the model name. * Currently "openssl", "nss" and "gcrypt" are supported. * Setting to "none" will disable crypto. * * crypto_cipher_type * should contain the cipher algo name. * It can be set to "none" to disable * encryption. * Currently supported by "nss" model: * "aes128", "aes192" and "aes256". * "openssl" model supports more modes and it strictly * depends on the openssl build. See: EVP_get_cipherbyname * openssl API call for details. * * crypto_hash_type * should contain the hashing algo name. * It can be set to "none" to disable * hashing. * Currently supported by "nss" model: * "md5", "sha1", "sha256", "sha384" and "sha512". * "openssl" model supports more modes and it strictly * depends on the openssl build. See: EVP_get_digestbyname * openssl API call for details. * * private_key will contain the private shared key. * It has to be at least KNET_MIN_KEY_LEN long. * * private_key_len * length of the provided private_key. * * config_num - knet supports 2 concurrent sets of crypto configurations, * to allow runtime change of crypto config and keys. * On RX both configurations will be used sequentially * in an attempt to decrypt/validate a packet (when 2 are available). * Note that this might slow down performance during a reconfiguration. * See also knet_handle_crypto_rx_clear_traffic(3) to enable / disable * processing of clear (unencrypted) traffic. * For TX, the user needs to specify which configuration to use via * knet_handle_crypto_use_config(3). * config_num accepts 0, 1 or 2 as the value. 0 should be used when * all crypto is being disabled. * Calling knet_handle_crypto_set_config(3) twice with * the same config_num will REPLACE the configuration and * NOT activate the second key. If the configuration is currently in use * EBUSY will be returned. See also knet_handle_crypto_use_config(3). * The correct sequence to perform a runtime rekey / reconfiguration * is: * - knet_handle_crypto_set_config(..., 1). -> first time config, will use config1 * - knet_handle_crypto_use_config(..., 1). -> switch TX to config 1 * - knet_handle_crypto_set_config(..., 2). -> install config2 and use it only for RX * - knet_handle_crypto_use_config(..., 2). -> switch TX to config 2 * - knet_handle_crypto_set_config(..., 1). -> with a "none"/"none"/"none" configuration to * release the resources previously allocated * The application is responsible for synchronizing calls on the nodes * to make sure the new config is in place before switching the TX configuration. * Failure to do so will result in knet being unable to talk to some of the nodes. * * Implementation notes/current limitations: * - enabling crypto, will increase latency as packets have * to processed. * - enabling crypto might reduce the overall throughtput * due to crypto data overhead. * - private/public key encryption/hashing is not currently * planned. * - crypto key must be the same for all hosts in the same * knet instance / configX. * - it is safe to call knet_handle_crypto_set_config multiple times at runtime. * The last config will be used. * IMPORTANT: a call to knet_handle_crypto_set_config can fail due to: * 1) failure to obtain locking * 2) errors to initializing the crypto level. * This can happen even in subsequent calls to knet_handle_crypto_set_config(3). * A failure in crypto init will restore the previous crypto configuration if any. * * @return * knet_handle_crypto_set_config returns: * @retval 0 on success * @retval -1 on error and errno is set. * @retval -2 on crypto subsystem initialization error. No errno is provided at the moment (yet). */ int knet_handle_crypto_set_config(knet_handle_t knet_h, struct knet_handle_crypto_cfg *knet_handle_crypto_cfg, uint8_t config_num); #define KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC 0 #define KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC 1 /** * knet_handle_crypto_rx_clear_traffic * * @brief enable or disable RX processing of clear (unencrypted) traffic * * knet_h - pointer to knet_handle_t * * value - KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC or KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC * * @return * knet_handle_crypto_use_config returns: * @retval 0 on success * @retval -1 on error and errno is set. */ int knet_handle_crypto_rx_clear_traffic(knet_handle_t knet_h, uint8_t value); /** * knet_handle_crypto_use_config * * @brief specify crypto configuration to use for TX * * knet_h - pointer to knet_handle_t * * config_num - 1|2 use configuration 1 or 2, 0 for clear (unencrypted) traffic. * * @return * knet_handle_crypto_use_config returns: * @retval 0 on success * @retval -1 on error and errno is set. */ int knet_handle_crypto_use_config(knet_handle_t knet_h, uint8_t config_num); #define KNET_COMPRESS_THRESHOLD 100 /** * Structure passed into knet_handle_compress() * to tell knet what type of compression to use * for this communiction */ struct knet_handle_compress_cfg { /** Compression library to use, bzip2 etc... */ char compress_model[16]; /** Threshold. Packets smaller than this will not be compressed */ uint32_t compress_threshold; /** Passed into the compression library as an indication of the level of compression to apply */ int compress_level; }; /** * knet_handle_compress * * @brief Set up packet compression * * knet_h - pointer to knet_handle_t * * knet_handle_compress_cfg - * pointer to a knet_handle_compress_cfg structure * * compress_model contains the model name. * See "compress_level" for the list of accepted values. * Setting the value to "none" disables compression. * * compress_threshold * tells the transmission thread to NOT compress * any packets that are smaller than the value * indicated. Default 100 bytes. * Set to 0 to reset to the default. * Set to 1 to compress everything. * Max accepted value is KNET_MAX_PACKET_SIZE. * * compress_level is the "level" parameter for most models: * zlib: 0 (no compression), 1 (minimal) .. 9 (max compression). * lz4: 1 (max compression)... 9 (fastest compression). * lz4hc: 1 (min compression) ... LZ4HC_MAX_CLEVEL (16) or LZ4HC_CLEVEL_MAX (12) * depending on the version of lz4hc libknet was built with. * lzma: 0 (minimal) .. 9 (max compression) * bzip2: 1 (minimal) .. 9 (max compression) * For lzo2 it selects the algorithm to use: * 1 : lzo1x_1_compress (default) * 11 : lzo1x_1_11_compress * 12 : lzo1x_1_12_compress * 15 : lzo1x_1_15_compress * 999: lzo1x_999_compress * Other values select the default algorithm. * Please refer to the documentation of the respective * compression library for guidance about setting this * value. * * Implementation notes: * - it is possible to enable/disable compression at any time. * - nodes can be using a different compression algorithm at any time. * - knet does NOT implement the compression algorithm directly. it relies * on external libraries for this functionality. Please read * the libraries man pages to figure out which algorithm/compression * level is best for the data you are planning to transmit. * * @return * knet_handle_compress returns * 0 on success * -1 on error and errno is set. EINVAL means that either the model or the * level are not supported. */ int knet_handle_compress(knet_handle_t knet_h, struct knet_handle_compress_cfg *knet_handle_compress_cfg); /** * Detailed stats for this knet handle as returned by knet_handle_get_stats() */ struct knet_handle_stats { /** Size of the structure. set this to sizeof(struct knet_handle_stats) before calling */ size_t size; /** Number of uncompressed packets sent */ uint64_t tx_uncompressed_packets; /** Number of compressed packets sent */ uint64_t tx_compressed_packets; /** Number of bytes sent (as if uncompressed, ie actual data bytes) */ uint64_t tx_compressed_original_bytes; /** Number of bytes sent on the wire after compression */ uint64_t tx_compressed_size_bytes; /** Average(mean) time take to compress transmitted packets */ uint64_t tx_compress_time_ave; /** Minimum time taken to compress transmitted packets */ uint64_t tx_compress_time_min; /** Maximum time taken to compress transmitted packets */ uint64_t tx_compress_time_max; /** Number of times the compression attempt failed for some reason */ uint64_t tx_failed_to_compress; /** Number of packets where the compressed size was no smaller than the original */ uint64_t tx_unable_to_compress; /** Number of compressed packets received */ uint64_t rx_compressed_packets; /** Number of bytes received - after decompression */ uint64_t rx_compressed_original_bytes; /** Number of compressed bytes received before decompression */ uint64_t rx_compressed_size_bytes; /** Average(mean) time take to decompress received packets */ uint64_t rx_compress_time_ave; /** Minimum time take to decompress received packets */ uint64_t rx_compress_time_min; /** Maximum time take to decompress received packets */ uint64_t rx_compress_time_max; /** Number of times decompression failed */ uint64_t rx_failed_to_decompress; /** Number of encrypted packets sent */ uint64_t tx_crypt_packets; /** Cumulative byte overhead of encrypted traffic */ uint64_t tx_crypt_byte_overhead; /** Average(mean) time take to encrypt packets in usecs */ uint64_t tx_crypt_time_ave; /** Minimum time take to encrypto packets in usecs */ uint64_t tx_crypt_time_min; /** Maximum time take to encrypto packets in usecs */ uint64_t tx_crypt_time_max; /** Number of encrypted packets received */ uint64_t rx_crypt_packets; /** Average(mean) time take to decrypt received packets */ uint64_t rx_crypt_time_ave; /** Minimum time take to decrypt received packets in usecs */ uint64_t rx_crypt_time_min; /** Maximum time take to decrypt received packets in usecs */ uint64_t rx_crypt_time_max; }; /** * knet_handle_get_stats * * @brief Get statistics for compression & crypto * * knet_h - pointer to knet_handle_t * * knet_handle_stats * pointer to a knet_handle_stats structure * * struct_size * size of knet_handle_stats structure to allow * for backwards compatibility. libknet will only * copy this much data into the stats structure * so that older callers will not get overflowed if * new fields are added. * * @return * 0 on success * -1 on error and errno is set. * */ int knet_handle_get_stats(knet_handle_t knet_h, struct knet_handle_stats *stats, size_t struct_size); /* * Tell knet_handle_clear_stats whether to clear just the handle stats * or all of them. */ #define KNET_CLEARSTATS_HANDLE_ONLY 1 #define KNET_CLEARSTATS_HANDLE_AND_LINK 2 /** * knet_handle_clear_stats * * @brief Clear knet stats, link and/or handle * * knet_h - pointer to knet_handle_t * * clear_option - Which stats to clear, must be one of * * KNET_CLEARSTATS_HANDLE_ONLY or * KNET_CLEARSTATS_HANDLE_AND_LINK * * @return * 0 on success * -1 on error and errno is set. * */ int knet_handle_clear_stats(knet_handle_t knet_h, int clear_option); /** * Structure returned from get_crypto_list() containing * information about the installed cryptographic systems */ struct knet_crypto_info { /** Name of the crypto library/ openssl, nss,etc .. */ const char *name; /** Properties - currently unused */ uint8_t properties; /** Currently unused padding */ char pad[256]; }; /** * knet_get_crypto_list * * @brief Get a list of supported crypto libraries * * crypto_list - array of struct knet_crypto_info * * If NULL then only the number of structs is returned in crypto_list_entries * to allow the caller to allocate sufficient space. * libknet does not allow more than 256 crypto methods at the moment. * it is safe to allocate 256 structs to avoid calling * knet_get_crypto_list twice. * * crypto_list_entries - returns the number of structs in crypto_list * * @return * knet_get_crypto_list returns * 0 on success * -1 on error and errno is set. */ int knet_get_crypto_list(struct knet_crypto_info *crypto_list, size_t *crypto_list_entries); /** * Structure returned from get_compress_list() containing * information about the installed compression systems */ struct knet_compress_info { /** Name of the compression type bzip2, lz4, etc.. */ const char *name; /** Properties - currently unused */ uint8_t properties; /** Currently unused padding */ char pad[256]; }; /** * knet_get_compress_list * * @brief Get a list of support compression types * * compress_list - array of struct knet_compress_info * * If NULL then only the number of structs is returned in compress_list_entries * to allow the caller to allocate sufficient space. * libknet does not allow more than 256 compress methods at the moment. * it is safe to allocate 256 structs to avoid calling * knet_get_compress_list twice. * * compress_list_entries - returns the number of structs in compress_list * * @return * knet_get_compress_list returns * 0 on success * -1 on error and errno is set. */ int knet_get_compress_list(struct knet_compress_info *compress_list, size_t *compress_list_entries); /** * knet_handle_enable_onwire_ver_notify * * @brief install a callback to receive onwire changes * * knet_h - pointer to knet_handle_t * * onwire_ver_notify_fn_private_data * void pointer to data that can be used to identify * the callback. * * onwire_ver_notify_fn * is a callback function that is invoked every time * an onwire version change is detected. * The function allows libknet to notify the user * of onwire version changes. * onwire_min_ver - minimum onwire version supported * onwire_max_ver - maximum onwire version supported * onwire_ver - currently onwire version in use * This function MUST NEVER block or add substantial delays. * * NOTE: the callback function will be invoked upon install to * immediately notify the user of the current configuration. * During startup, it is safer to use onwire_min_ver and * onwire_ver on subsequent calls. * * @return * knet_handle_enable_onwire_ver_notify returns * 0 on success * -1 on error and errno is set. */ int knet_handle_enable_onwire_ver_notify(knet_handle_t knet_h, void *onwire_ver_notify_fn_private_data, void (*onwire_ver_notify_fn) ( void *private_data, uint8_t onwire_min_ver, uint8_t onwire_max_ver, uint8_t onwire_ver)); /** * knet_handle_get_onwire_ver * * @brief get onwire protocol version information * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * onwire_min_ver - minimum onwire version supported by local node. * this value is set to 0 for remote nodes. * * onwire_max_ver - maximum onwire version supported by local or * remote node. * * onwire_ver - currently onwire version in use by local or * remote node. * * @return * knet_handle_get_onwire_ver returns * 0 on success * -1 on error and errno is set. */ int knet_handle_get_onwire_ver(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t *onwire_min_ver, uint8_t *onwire_max_ver, uint8_t *onwire_ver); /** * knet_handle_set_onwire_ver * * @brief force onwire protocol version * * knet_h - pointer to knet_handle_t * * onwire_ver - onwire version to use. * reset to 0 to allow knet to detect * automatically the highest version. * * @return * knet_handle_get_onwire_ver returns * 0 on success * -1 on error and errno is set. */ int knet_handle_set_onwire_ver(knet_handle_t knet_h, uint8_t onwire_ver); /* * defrag buffer configuration defaults */ #define KNET_MIN_DEFRAG_BUFS_DEFAULT 32 #define KNET_MAX_DEFRAG_BUFS_DEFAULT 1024 #define KNET_SHRINK_THRESHOLD_DEFAULT 25 /** * reclaim_policy for defrag buffers */ typedef enum { RECLAIM_POLICY_AVERAGE = 0, RECLAIM_POLICY_ABSOLUTE = 1 /* default */ } defrag_bufs_reclaim_policy_t; /** * knet_handle_get_host_defrag_bufs * * @brief Return the defrag buffers parameters for hosts * * knet_h - pointer to knet_handle_t * * min_defrag_bufs - minimum defrag buffers for each host * * max_defrag_bufs - maximum defrag buffers for each host * * shrink_threshold - define buffer usage threshold in % * below which buffers will be shrunk. * This is measured over the last * usage_samples_timespan, as an * average of usage_samples. * * reclaim_policy - define how % threshold is calculated. * * @return * knet_handle_set_host_defrag_bufs returns * 0 on success * -1 on error and errno is set. */ int knet_handle_get_host_defrag_bufs(knet_handle_t knet_h, uint16_t *min_defrag_bufs, uint16_t *max_defrag_bufs, uint8_t *shrink_threshold, defrag_bufs_reclaim_policy_t *reclaim_policy); /** * knet_handle_set_host_defrag_bufs * * @brief configure defrag buffers parameters per host * * knet_h - pointer to knet_handle_t * * min_defrag_bufs - minimum defrag buffers for each host, * This should be a power of 2. * * max_defrag_bufs - maximum defrag buffers for each host, * This should be a power of 2. * * shrink_threshold - define buffer usage threshold in % * below which buffers will be shrunk. * This is measured over the last * usage_samples_timespan, as an * average of usage_samples. * Only values less than or equal to 50% are accepted. * * reclaim_policy - define how % threshold is calculated. * * @note The defrag buffer parameters are global to a handle * but the buffers themselves are allocated and shrunk per-host. * * @return * knet_handle_set_host_defrag_bufs returns * 0 on success * -1 on error and errno is set. */ int knet_handle_set_host_defrag_bufs(knet_handle_t knet_h, uint16_t min_defrag_bufs, uint16_t max_defrag_bufs, uint8_t shrink_threshold, defrag_bufs_reclaim_policy_t reclaim_policy); /* * host structs/API calls */ /** * knet_host_add * * @brief Add a new host ID to knet * * knet_h - pointer to knet_handle_t * * host_id - each host in a knet is identified with a unique ID * (see also knet_handle_new(3)) * * @return * knet_host_add returns: * 0 on success * -1 on error and errno is set. */ int knet_host_add(knet_handle_t knet_h, knet_node_id_t host_id); /** * knet_host_remove * * @brief Remove a host ID from knet * * knet_h - pointer to knet_handle_t * * host_id - each host in a knet is identified with a unique ID * (see also knet_handle_new(3)) * * @return * knet_host_remove returns: * 0 on success * -1 on error and errno is set. */ int knet_host_remove(knet_handle_t knet_h, knet_node_id_t host_id); /** * knet_host_set_name * * @brief Set the name of a knet host * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * name - this name will be used for pretty logging and eventually * search for hosts (see also knet_handle_host_get_name(2) and knet_handle_host_get_id(3)). * Only up to KNET_MAX_HOST_LEN - 1 bytes will be accepted and * name has to be unique for each host. * * @return * knet_host_set_name returns: * 0 on success * -1 on error and errno is set. */ int knet_host_set_name(knet_handle_t knet_h, knet_node_id_t host_id, const char *name); /** * knet_host_get_name_by_host_id * * @brief Get the name of a host given its ID * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * name - pointer to a preallocated buffer of at least size KNET_MAX_HOST_LEN * where the current host name will be stored * (as set by knet_host_set_name or default by knet_host_add) * * @return * knet_host_get_name_by_host_id returns: * 0 on success * -1 on error and errno is set (name is left untouched) */ int knet_host_get_name_by_host_id(knet_handle_t knet_h, knet_node_id_t host_id, char *name); /** * knet_host_get_id_by_host_name * * @brief Get the ID of a host given its name * * knet_h - pointer to knet_handle_t * * name - name to lookup, max len KNET_MAX_HOST_LEN * * host_id - where to store the result * * @return * knet_host_get_id_by_host_name returns: * 0 on success * -1 on error and errno is set. */ int knet_host_get_id_by_host_name(knet_handle_t knet_h, const char *name, knet_node_id_t *host_id); /** * knet_host_get_host_list * * @brief Get a list of hosts known to knet * * knet_h - pointer to knet_handle_t * * host_ids - array of at lest KNET_MAX_HOST size * * host_ids_entries - * number of entries writted in host_ids * * @return * knet_host_get_host_list returns * 0 on success * -1 on error and errno is set. */ int knet_host_get_host_list(knet_handle_t knet_h, knet_node_id_t *host_ids, size_t *host_ids_entries); /* * define switching policies */ #define KNET_LINK_POLICY_PASSIVE 0 #define KNET_LINK_POLICY_ACTIVE 1 #define KNET_LINK_POLICY_RR 2 /** * knet_host_set_policy * * @brief Set the switching policy for a host's links * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * policy - there are currently 3 kind of simple switching policies * based on link configuration. * KNET_LINK_POLICY_PASSIVE - the active link with the highest * priority (highest number) will be used. * if one or more active links share * the same priority, the one with * lowest link_id will be used. * * KNET_LINK_POLICY_ACTIVE - all active links will be used * simultaneously to send traffic. * link priority is ignored. * * KNET_LINK_POLICY_RR - round-robin policy, every packet * will be send on a different active * link. * * @return * knet_host_set_policy returns * 0 on success * -1 on error and errno is set. */ int knet_host_set_policy(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t policy); /** * knet_host_get_policy * * @brief Get the switching policy for a host's links * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * policy - will contain the current configured switching policy. * Default is passive when creating a new host. * * @return * knet_host_get_policy returns * 0 on success * -1 on error and errno is set. */ int knet_host_get_policy(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t *policy); /** * knet_host_enable_status_change_notify * * @brief Install a callback to get host status change events * * knet_h - pointer to knet_handle_t * * host_status_change_notify_fn_private_data - * void pointer to data that can be used to identify * the callback * * host_status_change_notify_fn - * is a callback function that is invoked every time * there is a change in the host status. * host status is identified by: * - reachable, this host can send/receive data to/from host_id * - remote, 0 if the host_id is connected locally or 1 if * the there is one or more knet host(s) in between. * NOTE: re-switching is NOT currently implemented, * but this is ready for future and can avoid * an API/ABI breakage later on. * - external, 0 if the host_id is configured locally or 1 if * it has been added from remote nodes config. * NOTE: dynamic topology is NOT currently implemented, * but this is ready for future and can avoid * an API/ABI breakage later on. * This function MUST NEVER block or add substantial delays. * * @return * knet_host_status_change_notify returns * 0 on success * -1 on error and errno is set. */ int knet_host_enable_status_change_notify(knet_handle_t knet_h, void *host_status_change_notify_fn_private_data, void (*host_status_change_notify_fn) ( void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)); /* * define host status structure for quick lookup * struct is in flux as more stats will be added soon * * reachable host_id can be seen either directly connected * or via another host_id * * remote 0 = node is connected locally, 1 is visible via * via another host_id * * external 0 = node is configured/known locally, * 1 host_id has been received via another host_id */ /** * status of a knet host, returned from knet_host_get_status() */ struct knet_host_status { /** Whether the host is currently reachable */ uint8_t reachable; /** Whether the host is a remote node (not currently implemented) */ uint8_t remote; /** Whether the host is external (not currently implemented) */ uint8_t external; /* add host statistics */ }; /** * knet_host_get_status * * @brief Get the status of a host * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * status - pointer to knet_host_status struct * * @return * knet_handle_pmtud_get returns * 0 on success * -1 on error and errno is set. */ int knet_host_get_status(knet_handle_t knet_h, knet_node_id_t host_id, struct knet_host_status *status); /* * link structs/API calls * * every host allocated/managed by knet_host_* has * KNET_MAX_LINK structures to define the network * paths that connect 2 hosts. * * Each link is identified by a link_id that has a * values between 0 and KNET_MAX_LINK - 1. * * KNOWN LIMITATIONS: * * - let's assume the scenario where two hosts are connected * with any number of links. link_id must match on both sides. * If host_id 0 link_id 0 is configured to connect IP1 to IP2 and * host_id 0 link_id 1 is configured to connect IP3 to IP4, * host_id 1 link_id 0 _must_ connect IP2 to IP1 and likewise * host_id 1 link_id 1 _must_ connect IP4 to IP3. * We might be able to lift this restriction in future, by using * other data to determine src/dst link_id, but for now, deal with it. */ /* * commodity functions to convert strings to sockaddr and viceversa */ /** * knet_strtoaddr * * @brief Convert a hostname string to an address * * host - IPaddr/hostname to convert * be aware only the first IP address will be returned * in case a hostname resolves to multiple IP * * port - port to connect to * * ss - sockaddr_storage where to store the converted data * * sslen - len of the sockaddr_storage * * @return * knet_strtoaddr returns same error codes as getaddrinfo * */ int knet_strtoaddr(const char *host, const char *port, struct sockaddr_storage *ss, socklen_t sslen); /** * knet_addrtostr * * @brief Convert an address to a host name * * ss - sockaddr_storage to convert * * sslen - len of the sockaddr_storage * * host - IPaddr/hostname where to store data * (recommended size: KNET_MAX_HOST_LEN) * * port - port buffer where to store data * (recommended size: KNET_MAX_PORT_LEN) * * @return * knet_strtoaddr returns same error codes as getnameinfo */ int knet_addrtostr(const struct sockaddr_storage *ss, socklen_t sslen, char *addr_buf, size_t addr_buf_size, char *port_buf, size_t port_buf_size); #define KNET_TRANSPORT_LOOPBACK 0 #define KNET_TRANSPORT_UDP 1 -#define KNET_TRANSPORT_SCTP 2 #define KNET_MAX_TRANSPORTS UINT8_MAX /* * The Loopback transport is only valid for connections to localhost, the host * with the same node_id specified in knet_handle_new(). Only one link of this * type is allowed. Data sent down a LOOPBACK link will be copied directly from * the knet send datafd to the knet receive datafd so the application must be set * up to take data from that socket at least as often as it is sent or deadlocks * could occur. If used, a LOOPBACK link must be the only link configured to the * local host. */ /** * Transport information returned from knet_get_transport_list() */ struct knet_transport_info { - /** Transport name. UDP, SCTP, etc... */ + /** Transport name. UDP, etc... */ const char *name; /** value that can be used for knet_link_set_config() */ uint8_t id; /** currently unused */ uint8_t properties; /** currently unused */ char pad[256]; }; /** * knet_get_transport_list * * @brief Get a list of the transports support by this build of knet * * transport_list - an array of struct transport_info that must be * at least of size struct transport_info * KNET_MAX_TRANSPORTS * * transport_list_entries - pointer to a size_t where to store how many transports * are available in this build of libknet. * * @return * knet_get_transport_list returns * 0 on success * -1 on error and errno is set. */ int knet_get_transport_list(struct knet_transport_info *transport_list, size_t *transport_list_entries); /** * knet_get_transport_name_by_id * * @brief Get a transport name from its ID number * * transport - one of the KNET_TRANSPORT_xxx constants * * @return * knet_get_transport_name_by_id returns: * * @retval pointer to the name on success or * @retval NULL on error and errno is set. */ const char *knet_get_transport_name_by_id(uint8_t transport); /** * knet_get_transport_id_by_name * * @brief Get a transport ID from its name * - * name - transport name (UDP/SCTP/etc) + * name - transport name (UDP/etc) * * @return * knet_get_transport_name_by_id returns: * * @retval KNET_MAX_TRANSPORTS on error and errno is set accordingly * @retval KNET_TRANSPORT_xxx on success. */ uint8_t knet_get_transport_id_by_name(const char *name); #define KNET_TRANSPORT_DEFAULT_RECONNECT_INTERVAL 1000 /** * knet_handle_set_transport_reconnect_interval * * @brief Set the interval between transport attempts to reconnect a failed link * * knet_h - pointer to knet_handle_t * * msecs - milliseconds * * @return * knet_handle_set_transport_reconnect_interval returns * 0 on success * -1 on error and errno is set. */ int knet_handle_set_transport_reconnect_interval(knet_handle_t knet_h, uint32_t msecs); /** * knet_handle_get_transport_reconnect_interval * * @brief Get the interval between transport attempts to reconnect a failed link * * knet_h - pointer to knet_handle_t * * msecs - milliseconds * * @return * knet_handle_get_transport_reconnect_interval returns * 0 on success * -1 on error and errno is set. */ int knet_handle_get_transport_reconnect_interval(knet_handle_t knet_h, uint32_t *msecs); /** * knet_link_set_config * * @brief Configure the link to a host * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * transport - one of the KNET_TRANSPORT_xxx constants * * src_addr - sockaddr_storage that can be either IPv4 or IPv6 * * dst_addr - sockaddr_storage that can be either IPv4 or IPv6 * this can be null if we don't know the incoming * IP address/port and the link will remain quiet * till the node on the other end will initiate a * connection * * flags - KNET_LINK_FLAG_* * * @return * knet_link_set_config returns * 0 on success * -1 on error and errno is set. */ int knet_link_set_config(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t transport, struct sockaddr_storage *src_addr, struct sockaddr_storage *dst_addr, uint64_t flags); /** * knet_link_get_config * * @brief Get the link configutation information * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * transport - see knet_link_set_config(3) * * src_addr - sockaddr_storage that can be either IPv4 or IPv6 * * dst_addr - sockaddr_storage that can be either IPv4 or IPv6 * * dynamic - 0 if dst_addr is static or 1 if dst_addr is dynamic. * In case of 1, dst_addr can be NULL and it will be left * untouched. * * flags - KNET_LINK_FLAG_* * * @return * knet_link_get_config returns * 0 on success. * -1 on error and errno is set. */ int knet_link_get_config(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t *transport, struct sockaddr_storage *src_addr, struct sockaddr_storage *dst_addr, uint8_t *dynamic, uint64_t *flags); /** * knet_link_clear_config * * @brief Clear link information and disconnect the link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * @return * knet_link_clear_config returns * 0 on success. * -1 on error and errno is set. */ int knet_link_clear_config(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id); /* * Access lists management for open links * see also knet_handle_enable_access_lists(3) */ /** * check_type_t * @brief address type enum for knet access lists * * CHECK_TYPE_ADDRESS is the equivalent of a single entry / IP address. * for example: 10.1.9.3 * and the entry is stored in ss1. ss2 can be NULL. * * CHECK_TYPE_MASK is used to configure network/netmask. * for example: 192.168.0.0/24 * the network is stored in ss1 and the netmask in ss2. * * CHECK_TYPE_RANGE defines a value / range of ip addresses. * for example: 172.16.0.1-172.16.0.10 * the start is stored in ss1 and the end in ss2. * * Please be aware that the above examples refer only to IP based protocols. * Other protocols might use ss1 and ss2 in slightly different ways. * At the moment knet only supports IP based protocol, though that might change * in the future. */ typedef enum { CHECK_TYPE_ADDRESS, CHECK_TYPE_MASK, CHECK_TYPE_RANGE } check_type_t; /** * check_acceptreject_t * * @brief enum for accept/reject in knet access lists * * accept or reject incoming packets defined in the access list entry */ typedef enum { CHECK_ACCEPT, CHECK_REJECT } check_acceptreject_t; /** * knet_link_add_acl * * @brief Add access list entry to an open link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * ss1 / ss2 / type / acceptreject - see typedef definitions for details * * IMPORTANT: the order in which access lists are added is critical and it * is left to the user to add them in the right order. knet * will not attempt to logically sort them. * * For example: * 1 - accept from 10.0.0.0/8 * 2 - reject from 10.0.0.1/32 * * is not the same as: * * 1 - reject from 10.0.0.1/32 * 2 - accept from 10.0.0.0/8 * * In the first example, rule number 2 will never match because * packets from 10.0.0.1 will be accepted by rule number 1. * * @return * knet_link_add_acl returns * 0 on success. * -1 on error and errno is set. */ int knet_link_add_acl(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, struct sockaddr_storage *ss1, struct sockaddr_storage *ss2, check_type_t type, check_acceptreject_t acceptreject); /** * knet_link_insert_acl * * @brief Insert access list entry to an open link at given index * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * index - insert at position "index" where 0 is the first entry and -1 * appends to the current list. * * ss1 / ss2 / type / acceptreject - see typedef definitions for details * * @return * knet_link_insert_acl returns * 0 on success. * -1 on error and errno is set. */ int knet_link_insert_acl(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, int index, struct sockaddr_storage *ss1, struct sockaddr_storage *ss2, check_type_t type, check_acceptreject_t acceptreject); /** * knet_link_rm_acl * * @brief Remove access list entry from an open link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * ss1 / ss2 / type / acceptreject - see typedef definitions for details * * IMPORTANT: the data passed to this API call must match exactly that passed * to knet_link_add_acl(3). * * @return * knet_link_rm_acl returns * 0 on success. * -1 on error and errno is set. */ int knet_link_rm_acl(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, struct sockaddr_storage *ss1, struct sockaddr_storage *ss2, check_type_t type, check_acceptreject_t acceptreject); /** * knet_link_clear_acl * * @brief Remove all access list entries from an open link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * @return * knet_link_clear_acl returns * 0 on success. * -1 on error and errno is set. */ int knet_link_clear_acl(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id); /** * knet_link_set_enable * * @brief Enable traffic on a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * enabled - 0 disable the link, 1 enable the link * * @return * knet_link_set_enable returns * 0 on success * -1 on error and errno is set. */ int knet_link_set_enable(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, unsigned int enabled); /** * knet_link_get_enable * * @brief Find out whether a link is enabled or not * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * enabled - 0 disable the link, 1 enable the link * * @return * knet_link_get_enable returns * 0 on success * -1 on error and errno is set. */ int knet_link_get_enable(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, unsigned int *enabled); #define KNET_LINK_DEFAULT_PING_INTERVAL 1000 /* 1 second */ #define KNET_LINK_DEFAULT_PING_TIMEOUT 2000 /* 2 seconds */ #define KNET_LINK_DEFAULT_PING_PRECISION 2048 /* samples */ /** * knet_link_set_ping_timers * * @brief Set the ping timers for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * interval - specify the ping interval in milliseconds. * * timeout - if no pong is received within this time, * the link is declared dead, in milliseconds. * NOTE: in future it will be possible to set timeout to 0 * for an autocalculated timeout based on interval, pong_count * and latency. The API already accept 0 as value and it will * return ENOSYS / -1. Once the automatic calculation feature * will be implemented, this call will only return EINVAL * for incorrect values. * * precision - how many values of latency are used to calculate * the average link latency (see also knet_link_get_status(3)) * * @return * knet_link_set_ping_timers returns * 0 on success * -1 on error and errno is set. */ int knet_link_set_ping_timers(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, time_t interval, time_t timeout, unsigned int precision); /** * knet_link_get_ping_timers * * @brief Get the ping timers for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * interval - ping interval * * timeout - if no pong is received within this time, * the link is declared dead * * precision - how many values of latency are used to calculate * the average link latency (see also knet_link_get_status(3)) * * @return * knet_link_get_ping_timers returns * 0 on success * -1 on error and errno is set. */ int knet_link_get_ping_timers(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, time_t *interval, time_t *timeout, unsigned int *precision); #define KNET_LINK_DEFAULT_PONG_COUNT 5 /** * knet_link_set_pong_count * * @brief Set the pong count for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * pong_count - how many valid ping/pongs before a link is marked UP. * default: 5, value should be > 0 * * @return * knet_link_set_pong_count returns * 0 on success * -1 on error and errno is set. */ int knet_link_set_pong_count(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t pong_count); /** * knet_link_get_pong_count * * @brief Get the pong count for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * pong_count - how many valid ping/pongs before a link is marked UP. * default: 5, value should be > 0 * * @return * knet_link_get_pong_count returns * 0 on success * -1 on error and errno is set. */ int knet_link_get_pong_count(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t *pong_count); /** * knet_link_set_priority * * @brief Set the priority for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * priority - specify the switching priority for this link * see also knet_host_set_policy * * @return * knet_link_set_priority returns * 0 on success * -1 on error and errno is set. */ int knet_link_set_priority(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t priority); /** * knet_link_get_priority * * @brief Get the priority for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * priority - gather the switching priority for this link * see also knet_host_set_policy * * @return * knet_link_get_priority returns * 0 on success * -1 on error and errno is set. */ int knet_link_get_priority(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t *priority); /** * knet_link_get_link_list * * @brief Get a list of links connecting a host * * knet_h - pointer to knet_handle_t * * link_ids - array of at lest KNET_MAX_LINK size * with the list of configured links for a certain host. * * link_ids_entries - * number of entries contained in link_ids * * @return * knet_link_get_link_list returns * 0 on success * -1 on error and errno is set. */ int knet_link_get_link_list(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t *link_ids, size_t *link_ids_entries); /* * define link status structure for quick lookup * * src/dst_{ipaddr,port} strings are filled by * getnameinfo(3) when configuring the link. * if the link is dynamic (see knet_link_set_config(3)) * dst_ipaddr/port will contain ipaddr/port of the currently * connected peer or "Unknown" if it was not possible * to determine the ipaddr/port at runtime. * * enabled see also knet_link_set/get_enable. * * connected the link is connected to a peer and ping/pong traffic * is flowing. * * dynconnected the link has dynamic ip on the other end, and * we can see the other host is sending pings to us. * * pong_last if the link is down, this value tells us how long * ago this link was active. A value of 0 means that the link * has never been active. * * knet_link_stats structure that contains details statistics for the link */ #define MAX_LINK_EVENTS 16 /** * Stats for a knet link * returned from knet_link_get_status() as part of a knet_link_status structure * link stats are 'onwire', ie they indicate the number of actual bytes/packets * sent including overheads, not just data packets. */ struct knet_link_stats { /** Number of data packets sent */ uint64_t tx_data_packets; /** Number of data packets received */ uint64_t rx_data_packets; /** Number of data bytes sent */ uint64_t tx_data_bytes; /** Number of data bytes received */ uint64_t rx_data_bytes; /** Number of ping packets sent */ uint64_t rx_ping_packets; /** Number of ping packets received */ uint64_t tx_ping_packets; /** Number of ping bytes sent */ uint64_t rx_ping_bytes; /** Number of ping bytes received */ uint64_t tx_ping_bytes; /** Number of pong packets sent */ uint64_t rx_pong_packets; /** Number of pong packets received */ uint64_t tx_pong_packets; /** Number of pong bytes sent */ uint64_t rx_pong_bytes; /** Number of pong bytes received */ uint64_t tx_pong_bytes; /** Number of pMTU packets sent */ uint64_t rx_pmtu_packets; /** Number of pMTU packets received */ uint64_t tx_pmtu_packets; /** Number of pMTU bytes sent */ uint64_t rx_pmtu_bytes; /** Number of pMTU bytes received */ uint64_t tx_pmtu_bytes; /* These are only filled in when requested ie. they are not collected in realtime */ /** Total of all packets sent */ uint64_t tx_total_packets; /** Total of all packets received */ uint64_t rx_total_packets; /** Total number of bytes sent */ uint64_t tx_total_bytes; /** Total number of bytes received */ uint64_t rx_total_bytes; /** Total number of errors that occurred while sending */ uint64_t tx_total_errors; /** Total number of retries that occurred while sending */ uint64_t tx_total_retries; /** Total number of errors that occurred while sending pMTU packets */ uint32_t tx_pmtu_errors; /** Total number of retries that occurred while sending pMTU packets */ uint32_t tx_pmtu_retries; /** Total number of errors that occurred while sending ping packets */ uint32_t tx_ping_errors; /** Total number of retries that occurred while sending ping packets */ uint32_t tx_ping_retries; /** Total number of errors that occurred while sending pong packets */ uint32_t tx_pong_errors; /** Total number of retries that occurred while sending pong packets */ uint32_t tx_pong_retries; /** Total number of errors that occurred while sending data packets */ uint32_t tx_data_errors; /** Total number of retries that occurred while sending data packets */ uint32_t tx_data_retries; /** Minimum latency measured in usecs */ uint32_t latency_min; /** Maximum latency measured in usecs */ uint32_t latency_max; /** Average(mean) latency measured in usecs */ uint32_t latency_ave; /** Number of samples used to calculate latency */ uint32_t latency_samples; /** How many times the link has gone down */ uint32_t down_count; /** How many times the link has come up */ uint32_t up_count; /** * A circular buffer of time_t structs collecting the history * of up events on this link. * The index indicates current/last event. * it is safe to walk back the history by decreasing the index */ time_t last_up_times[MAX_LINK_EVENTS]; /** * A circular buffer of time_t structs collecting the history * of down events on this link. * The index indicates current/last event. * it is safe to walk back the history by decreasing the index */ time_t last_down_times[MAX_LINK_EVENTS]; /** Index of last element in the last_up_times[] array */ int8_t last_up_time_index; /** Index of last element in the last_down_times[] array */ int8_t last_down_time_index; /* Always add new stats at the end */ }; /** * Status of a knet link as returned from knet_link_get_status() */ struct knet_link_status { /** Size of the structure for ABI checking, set this to sizeof(knet_link_status) before calling knet_link_get_status() */ size_t size; /** Local IP address as a string*/ char src_ipaddr[KNET_MAX_HOST_LEN]; /** Local IP port as a string */ char src_port[KNET_MAX_PORT_LEN]; /** Remote IP address as a string */ char dst_ipaddr[KNET_MAX_HOST_LEN]; /** Remote IP port as a string*/ char dst_port[KNET_MAX_PORT_LEN]; /** Link is configured and admin enabled for traffic */ uint8_t enabled; /** Link is connected for data (local view) */ uint8_t connected; /** Link has been activated by remote dynip */ uint8_t dynconnected; /** Timestamp of the past pong received */ struct timespec pong_last; /** Currently detected MTU on this link */ unsigned int mtu; /** * Contains the size of the IP protocol, knet headers and * crypto headers (if configured). This value is filled in * ONLY after the first PMTUd run on that given link, * and can change if link configuration or crypto configuration * changes at runtime. * WARNING: in general mtu + proto_overhead might or might * not match the output of ifconfig mtu due to crypto * requirements to pad packets to some specific boundaries. */ unsigned int proto_overhead; /** Link statistics */ struct knet_link_stats stats; }; /** * knet_link_get_status * * @brief Get the status (and statistics) for a link * * knet_h - pointer to knet_handle_t * * host_id - see knet_host_add(3) * * link_id - see knet_link_set_config(3) * * status - pointer to knet_link_status struct * * struct_size - max size of knet_link_status - allows library to * add fields without ABI change. Returned structure * will be truncated to this length and .size member * indicates the full size. * * @return * knet_link_get_status returns * 0 on success * -1 on error and errno is set. */ int knet_link_get_status(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, struct knet_link_status *status, size_t struct_size); /** * knet_link_enable_status_change_notify * * @brief Install a callback to get a link status change events * * knet_h - pointer to knet_handle_t * * host_status_change_notify_fn_private_data - * void pointer to data that can be used to identify * the callback * * host_status_change_notify_fn - * is a callback function that is invoked every time * there is a change in a link status. * host status is identified by: * - connected, 0 if the link has been disconnected, 1 if the link * is connected. * - remote, 0 if the host_id is connected locally or 1 if * the there is one or more knet host(s) in between. * NOTE: re-switching is NOT currently implemented, * but this is ready for future and can avoid * an API/ABI breakage later on. * - external, 0 if the host_id is configured locally or 1 if * it has been added from remote nodes config. * NOTE: dynamic topology is NOT currently implemented, * but this is ready for future and can avoid * an API/ABI breakage later on. * This function MUST NEVER block or add substantial delays. * * @return * knet_host_status_change_notify returns * 0 on success * -1 on error and errno is set. */ int knet_link_enable_status_change_notify(knet_handle_t knet_h, void *link_status_change_notify_fn_private_data, void (*link_status_change_notify_fn) ( void *private_data, knet_node_id_t host_id, uint8_t link_id, uint8_t connected, uint8_t remote, uint8_t external)); /* * logging structs/API calls */ /* * libknet is composed of several subsystems. In order * to easily distinguish log messages coming from different * places, each subsystem has its own ID. * * 0-19 config/management * 20-39 internal threads * 40-59 transports * 60-69 crypto implementations */ #define KNET_SUB_COMMON 0 /* common.c */ #define KNET_SUB_HANDLE 1 /* handle.c alloc/dealloc config changes */ #define KNET_SUB_HOST 2 /* host add/del/modify */ #define KNET_SUB_LISTENER 3 /* listeners add/del/modify... */ #define KNET_SUB_LINK 4 /* link add/del/modify */ #define KNET_SUB_TRANSPORT 5 /* Transport common */ #define KNET_SUB_CRYPTO 6 /* crypto.c config generic layer */ #define KNET_SUB_COMPRESS 7 /* compress.c config generic layer */ #define KNET_SUB_FILTER 19 /* allocated for users to log from dst_filter */ #define KNET_SUB_DSTCACHE 20 /* switching thread (destination cache handling) */ #define KNET_SUB_HEARTBEAT 21 /* heartbeat thread */ #define KNET_SUB_PMTUD 22 /* Path MTU Discovery thread */ #define KNET_SUB_TX 23 /* send to link thread */ #define KNET_SUB_RX 24 /* recv from link thread */ #define KNET_SUB_TRANSP_BASE 40 /* Base log level for transports */ #define KNET_SUB_TRANSP_LOOPBACK (KNET_SUB_TRANSP_BASE + KNET_TRANSPORT_LOOPBACK) #define KNET_SUB_TRANSP_UDP (KNET_SUB_TRANSP_BASE + KNET_TRANSPORT_UDP) -#define KNET_SUB_TRANSP_SCTP (KNET_SUB_TRANSP_BASE + KNET_TRANSPORT_SCTP) #define KNET_SUB_NSSCRYPTO 60 /* crypto_nss.c */ #define KNET_SUB_OPENSSLCRYPTO 61 /* crypto_openssl.c */ #define KNET_SUB_GCRYPTCRYPTO 62 /* crypto_gcrypt.c */ #define KNET_SUB_ZLIBCOMP 70 /* compress_zlib.c */ #define KNET_SUB_LZ4COMP 71 /* compress_lz4.c */ #define KNET_SUB_LZ4HCCOMP 72 /* compress_lz4.c */ #define KNET_SUB_LZO2COMP 73 /* compress_lzo.c */ #define KNET_SUB_LZMACOMP 74 /* compress_lzma.c */ #define KNET_SUB_BZIP2COMP 75 /* compress_bzip2.c */ #define KNET_SUB_ZSTDCOMP 76 /* compress_zstd.c */ #define KNET_SUB_UNKNOWN UINT8_MAX - 1 #define KNET_MAX_SUBSYSTEMS UINT8_MAX /* * Convert between subsystem IDs and names */ /** * knet_log_get_subsystem_name * * @brief Get a logging system name from its numeric ID * * @return * returns internal name of the subsystem or "common" */ const char *knet_log_get_subsystem_name(uint8_t subsystem); /** * knet_log_get_subsystem_id * * @brief Get a logging system ID from its name * * @return * returns internal ID of the subsystem or KNET_SUB_COMMON */ uint8_t knet_log_get_subsystem_id(const char *name); /* * 5 log levels are enough for everybody */ #define KNET_LOG_ERR 0 /* unrecoverable errors/conditions */ #define KNET_LOG_WARN 1 /* recoverable errors/conditions */ #define KNET_LOG_INFO 2 /* info, link up/down, config changes.. */ #define KNET_LOG_DEBUG 3 #define KNET_LOG_TRACE 4 /* * Convert between log level values and names */ /** * knet_log_get_loglevel_name * * @brief Get a logging level name from its numeric ID * * @return * returns internal name of the log level or "ERROR" for unknown values */ const char *knet_log_get_loglevel_name(uint8_t level); /** * knet_log_get_loglevel_id * * @brief Get a logging level ID from its name * * @return * returns internal log level ID or KNET_LOG_ERR for invalid names */ uint8_t knet_log_get_loglevel_id(const char *name); /* * every log message is composed by a text message * and message level/subsystem IDs. * In order to make debugging easier it is possible to send those packets * straight to stdout/stderr (see knet_bench.c stdout option). */ #define KNET_MAX_LOG_MSG_SIZE 254 #if KNET_MAX_LOG_MSG_SIZE > PIPE_BUF #error KNET_MAX_LOG_MSG_SIZE cannot be bigger than PIPE_BUF for guaranteed system atomic writes #endif /** * Structure of a log message sent to the logging fd */ struct knet_log_msg { /** Text of the log message */ char msg[KNET_MAX_LOG_MSG_SIZE]; /** Subsystem that sent this message. KNET_SUB_* */ uint8_t subsystem; /** Logging level of this message. KNET_LOG_* */ uint8_t msglevel; /** Pointer to the handle generating the log message */ knet_handle_t knet_h; }; /** * knet_log_set_loglevel * * @brief Set the logging level for a subsystem * * knet_h - same as above * * subsystem - same as above * * level - same as above * * knet_log_set_loglevel allows fine control of log levels by subsystem. * See also knet_handle_new for defaults. * * @return * knet_log_set_loglevel returns * 0 on success * -1 on error and errno is set. */ int knet_log_set_loglevel(knet_handle_t knet_h, uint8_t subsystem, uint8_t level); /** * knet_log_get_loglevel * * @brief Get the logging level for a subsystem * * knet_h - same as above * * subsystem - same as above * * level - same as above * * @return * knet_log_get_loglevel returns * 0 on success * -1 on error and errno is set. */ int knet_log_get_loglevel(knet_handle_t knet_h, uint8_t subsystem, uint8_t *level); #endif diff --git a/libknet/logging.c b/libknet/logging.c index 9466c71e..ae69e252 100644 --- a/libknet/logging.c +++ b/libknet/logging.c @@ -1,276 +1,275 @@ /* * Copyright (C) 2010-2025 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include #include "internals.h" #include "logging.h" #include "threads_common.h" static struct pretty_names subsystem_names[KNET_MAX_SUBSYSTEMS] = { { "common", KNET_SUB_COMMON }, { "handle", KNET_SUB_HANDLE }, { "host", KNET_SUB_HOST }, { "listener", KNET_SUB_LISTENER }, { "link", KNET_SUB_LINK }, { "transport", KNET_SUB_TRANSPORT }, { "crypto", KNET_SUB_CRYPTO }, { "compress", KNET_SUB_COMPRESS }, { "filter", KNET_SUB_FILTER }, { "dstcache", KNET_SUB_DSTCACHE }, { "heartbeat", KNET_SUB_HEARTBEAT }, { "pmtud", KNET_SUB_PMTUD }, { "tx", KNET_SUB_TX }, { "rx", KNET_SUB_RX }, { "loopback", KNET_SUB_TRANSP_LOOPBACK }, { "udp", KNET_SUB_TRANSP_UDP }, - { "sctp", KNET_SUB_TRANSP_SCTP }, { "nsscrypto", KNET_SUB_NSSCRYPTO }, { "opensslcrypto", KNET_SUB_OPENSSLCRYPTO }, { "gcryptcrypto", KNET_SUB_GCRYPTCRYPTO }, { "zlibcomp", KNET_SUB_ZLIBCOMP }, { "lz4comp", KNET_SUB_LZ4COMP }, { "lz4hccomp", KNET_SUB_LZ4HCCOMP }, { "lzo2comp", KNET_SUB_LZO2COMP }, { "lzmacomp", KNET_SUB_LZMACOMP }, { "bzip2comp", KNET_SUB_BZIP2COMP }, { "zstdcomp", KNET_SUB_ZSTDCOMP }, { "unknown", KNET_SUB_UNKNOWN } /* unknown MUST always be last in this array */ }; const char *knet_log_get_subsystem_name(uint8_t subsystem) { unsigned int i; for (i = 0; i < KNET_MAX_SUBSYSTEMS; i++) { if (subsystem_names[i].val == KNET_SUB_UNKNOWN) { break; } if (subsystem_names[i].val == subsystem) { errno = 0; return subsystem_names[i].name; } } return "unknown"; } uint8_t knet_log_get_subsystem_id(const char *name) { unsigned int i; for (i = 0; i < KNET_MAX_SUBSYSTEMS; i++) { if (subsystem_names[i].val == KNET_SUB_UNKNOWN) { break; } if (strcasecmp(name, subsystem_names[i].name) == 0) { errno = 0; return subsystem_names[i].val; } } return KNET_SUB_UNKNOWN; } static int is_valid_subsystem(uint8_t subsystem) { unsigned int i; for (i = 0; i < KNET_MAX_SUBSYSTEMS; i++) { if ((subsystem != KNET_SUB_UNKNOWN) && (subsystem_names[i].val == KNET_SUB_UNKNOWN)) { break; } if (subsystem_names[i].val == subsystem) { return 0; } } return -1; } static struct pretty_names loglevel_names[KNET_LOG_TRACE + 1] = { { "ERROR", KNET_LOG_ERR }, { "WARNING", KNET_LOG_WARN }, { "info", KNET_LOG_INFO }, { "debug", KNET_LOG_DEBUG }, { "trace", KNET_LOG_TRACE } }; const char *knet_log_get_loglevel_name(uint8_t level) { unsigned int i; for (i = 0; i <= KNET_LOG_TRACE; i++) { if (loglevel_names[i].val == level) { errno = 0; return loglevel_names[i].name; } } return "ERROR"; } uint8_t knet_log_get_loglevel_id(const char *name) { unsigned int i; for (i = 0; i <= KNET_LOG_TRACE; i++) { if (strcasecmp(name, loglevel_names[i].name) == 0) { errno = 0; return loglevel_names[i].val; } } return KNET_LOG_ERR; } int knet_log_set_loglevel(knet_handle_t knet_h, uint8_t subsystem, uint8_t level) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (is_valid_subsystem(subsystem) < 0) { errno = EINVAL; return -1; } if (level > KNET_LOG_TRACE) { errno = EINVAL; return -1; } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, subsystem, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } knet_h->log_levels[subsystem] = level; pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } int knet_log_get_loglevel(knet_handle_t knet_h, uint8_t subsystem, uint8_t *level) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (is_valid_subsystem(subsystem) < 0) { errno = EINVAL; return -1; } if (!level) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, subsystem, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } *level = knet_h->log_levels[subsystem]; pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } void log_msg(knet_handle_t knet_h, uint8_t subsystem, uint8_t msglevel, const char *fmt, ...) { va_list ap; struct knet_log_msg msg; size_t byte_cnt = 0; int len; int retry_loop = 0; if ((!knet_h) || (subsystem == KNET_MAX_SUBSYSTEMS) || (msglevel > knet_h->log_levels[subsystem])) return; if (knet_h->logfd <= 0) goto out; memset(&msg, 0, sizeof(struct knet_log_msg)); msg.subsystem = subsystem; msg.msglevel = msglevel; msg.knet_h = knet_h; va_start(ap, fmt); #ifdef __clang__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wformat-nonliteral" #endif vsnprintf(msg.msg, sizeof(msg.msg), fmt, ap); #ifdef __clang__ #pragma clang diagnostic pop #endif va_end(ap); retry: while (byte_cnt < sizeof(struct knet_log_msg)) { len = write(knet_h->logfd, &msg, sizeof(struct knet_log_msg) - byte_cnt); if (len <= 0) { if (errno == EAGAIN) { struct timeval tv; /* * those 3 lines are the equivalent of usleep(1) * but usleep makes some static code analizers very * unhappy. * * this version is somewhat stolen from gnulib * nanosleep implementation */ tv.tv_sec = 0; tv.tv_usec = 1; select(0, NULL, NULL, NULL, &tv); retry_loop++; /* * arbitrary amount of retries. * tested with fun_log_bench, 10 retries was never hit */ if (retry_loop >= 100) { goto out; } goto retry; } goto out; } byte_cnt += len; } out: return; } diff --git a/libknet/onwire.c b/libknet/onwire.c index 560aa0c3..a08b2d56 100644 --- a/libknet/onwire.c +++ b/libknet/onwire.c @@ -1,265 +1,263 @@ /* * Copyright (C) 2019-2025 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under LGPL-2.0+ */ #include "config.h" #include #include #include #include "crypto.h" #include "internals.h" #include "logging.h" #include "common.h" #include "transport_udp.h" -#include "transport_sctp.h" /* * unencrypted packet looks like: * * | ip | protocol | knet_header | unencrypted data | * | onwire_len | * | proto_overhead | * | data_len | * | app MTU | * * encrypted packet looks like (not to scale): * * | ip | protocol | salt | crypto(knet_header | data) | crypto_data_pad | hash | * | onwire_len | * | proto_overhead | * | data_len | * | app MTU | * * knet_h->sec_block_size is >= 0 if encryption will pad the data * knet_h->sec_salt_size is >= 0 if encryption is enabled * knet_h->sec_hash_size is >= 0 if signing is enabled */ /* * this function takes in the data that we would like to send * and tells us the outgoing onwire data size with crypto and * all the headers adjustment. * calling thread needs to account for protocol overhead. */ size_t calc_data_outlen(knet_handle_t knet_h, size_t inlen) { size_t outlen = inlen, pad_len = 0; if (knet_h->sec_block_size) { /* * if the crypto mechanism requires padding, calculate the padding * and add it back to outlen because that's what the crypto layer * would do. */ pad_len = knet_h->sec_block_size - (outlen % knet_h->sec_block_size); outlen = outlen + pad_len; } return outlen + knet_h->sec_salt_size + knet_h->sec_hash_size; } /* * this function takes in the data that we would like to send * and tells us what is the real maximum data we can send * accounting for headers and crypto * calling thread needs to account for protocol overhead. */ size_t calc_max_data_outlen(knet_handle_t knet_h, size_t inlen) { size_t outlen = inlen, pad_len = 0; if (knet_h->sec_block_size) { /* * drop both salt and hash, that leaves only the crypto data and padding * we need to calculate the padding based on the real encrypted data * that includes the knet_header. */ outlen = outlen - (knet_h->sec_salt_size + knet_h->sec_hash_size); /* * if the crypto mechanism requires padding, calculate the padding * and remove it, to align the data. * NOTE: we need to remove pad_len + 1 because, based on testing, * if we send data that are already aligned to block_size, the * crypto implementations will add another block_size! * so we want to make sure that our data won't add an unnecessary * block_size that we need to remove later. */ pad_len = outlen % knet_h->sec_block_size; outlen = outlen - (pad_len + 1); /* * add both hash and salt size back, similar to padding above, * the crypto layer will add them to the outlen */ outlen = outlen + (knet_h->sec_salt_size + knet_h->sec_hash_size); } /* * drop KNET_HEADER_ALL_SIZE to provide a clean application MTU * and various crypto headers */ outlen = outlen - (KNET_HEADER_ALL_SIZE + knet_h->sec_salt_size + knet_h->sec_hash_size); return outlen; } /* * set the lowest possible value as failsafe for all links. * KNET_PMTUD_MIN_MTU_V4 < KNET_PMTUD_MIN_MTU_V6 * KNET_PMTUD_OVERHEAD_V6 > KNET_PMTUD_OVERHEAD_V4 - * KNET_PMTUD_SCTP_OVERHEAD > KNET_PMTUD_UDP_OVERHEAD */ size_t calc_min_mtu(knet_handle_t knet_h) { - return calc_max_data_outlen(knet_h, KNET_PMTUD_MIN_MTU_V4 - (KNET_PMTUD_OVERHEAD_V6 + KNET_PMTUD_SCTP_OVERHEAD)); + return calc_max_data_outlen(knet_h, KNET_PMTUD_MIN_MTU_V4 - (KNET_PMTUD_OVERHEAD_V6)); } int knet_handle_enable_onwire_ver_notify(knet_handle_t knet_h, void *onwire_ver_notify_fn_private_data, void (*onwire_ver_notify_fn) ( void *private_data, uint8_t onwire_min_ver, uint8_t onwire_max_ver, uint8_t onwire_ver)) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } knet_h->onwire_ver_notify_fn_private_data = onwire_ver_notify_fn_private_data; knet_h->onwire_ver_notify_fn = onwire_ver_notify_fn; if (knet_h->onwire_ver_notify_fn) { log_debug(knet_h, KNET_SUB_HANDLE, "onwire_ver_notify_fn enabled"); /* * generate an artificial call to notify the app of what´s curently * happening */ knet_h->onwire_ver_notify_fn(knet_h->onwire_ver_notify_fn_private_data, knet_h->onwire_min_ver, knet_h->onwire_max_ver, knet_h->onwire_ver); } else { log_debug(knet_h, KNET_SUB_HANDLE, "onwire_ver_notify_fn disabled"); } pthread_rwlock_unlock(&knet_h->global_rwlock); return 0; } int knet_handle_get_onwire_ver(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t *onwire_min_ver, uint8_t *onwire_max_ver, uint8_t *onwire_ver) { int err = 0, savederrno = 0; struct knet_host *host; if (!_is_valid_handle(knet_h)) { return -1; } if (!onwire_min_ver) { errno = EINVAL; return -1; } if (!onwire_max_ver) { errno = EINVAL; return -1; } if (!onwire_ver) { errno = EINVAL; return -1; } /* * we need a write lock here so that gathering host onwire info * is not racy (updated by thread_rx) and we can save a mutex_lock * to gather local node info. */ savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (host_id == knet_h->host_id) { *onwire_min_ver = knet_h->onwire_min_ver; *onwire_max_ver = knet_h->onwire_max_ver; *onwire_ver = knet_h->onwire_ver; } else { host = knet_h->host_index[host_id]; if (!host) { err = -1; savederrno = EINVAL; log_err(knet_h, KNET_SUB_HANDLE, "Unable to find host %u: %s", host_id, strerror(savederrno)); goto out_unlock; } *onwire_min_ver = 0; *onwire_max_ver = host->onwire_max_ver; *onwire_ver = host->onwire_ver; } out_unlock: pthread_rwlock_unlock(&knet_h->global_rwlock); errno = savederrno; return err; } int knet_handle_set_onwire_ver(knet_handle_t knet_h, uint8_t onwire_ver) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if ((onwire_ver) && ((onwire_ver < knet_h->onwire_min_ver) || (onwire_ver > knet_h->onwire_max_ver))) { errno = EINVAL; return -1; } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } knet_h->onwire_force_ver = onwire_ver; pthread_rwlock_unlock(&knet_h->global_rwlock); return 0; } diff --git a/libknet/tests/Makefile.am b/libknet/tests/Makefile.am index 7bf1b4f6..d141a6aa 100644 --- a/libknet/tests/Makefile.am +++ b/libknet/tests/Makefile.am @@ -1,126 +1,125 @@ # # Copyright (C) 2016-2025 Red Hat, Inc. All rights reserved. # # Authors: Fabio M. Di Nitto # # This software licensed under GPL-2.0+ # MAINTAINERCLEANFILES = Makefile.in include $(top_srcdir)/build-aux/check.mk include $(top_srcdir)/libknet/tests/api-check.mk EXTRA_DIST = \ api-test-coverage \ api-check.mk AM_CPPFLAGS = -I$(top_srcdir)/libknet $(XOPEN_CPPFLAGS) AM_CFLAGS += $(PTHREAD_CFLAGS) $(libqb_CFLAGS) $(zlib_CFLAGS) LIBS = $(top_builddir)/libknet/libknet.la \ $(PTHREAD_LIBS) $(dl_LIBS) $(zlib_LIBS) noinst_HEADERS = \ test-common.h # the order of those tests is NOT random. # some functions can only be tested properly after some dependents # API have been validated upfront. check_PROGRAMS = \ $(api_checks) \ $(int_checks) if RUN_FUN_TESTS check_PROGRAMS += $(fun_checks) endif int_checks = \ int_links_acl_ip_test \ int_timediff_test fun_checks = \ fun_config_crypto_test \ fun_onwire_upgrade_test \ fun_acl_check_test # checks below need to be executed manually # or with a specific environment long_run_checks = \ fun_pmtud_crypto_test benchmarks = \ knet_bench_test noinst_PROGRAMS = \ api_knet_handle_new_limit_test \ pckt_test \ $(benchmarks) \ $(long_run_checks) \ $(api_checks) \ $(int_checks) \ $(fun_checks) noinst_SCRIPTS = \ api-test-coverage TESTS = $(check_PROGRAMS) if INSTALL_TESTS testsuitedir = $(TESTDIR) testsuite_PROGRAMS = $(noinst_PROGRAMS) endif check-local: check-api-test-coverage check-annocheck-bins check-api-test-coverage: chmod u+x $(top_srcdir)/libknet/tests/api-test-coverage $(top_srcdir)/libknet/tests/api-test-coverage $(top_srcdir) $(top_builddir) pckt_test_SOURCES = pckt_test.c int_links_acl_ip_test_SOURCES = int_links_acl_ip.c \ ../common.c \ ../compat.c \ ../logging.c \ ../netutils.c \ ../threads_common.c \ ../onwire.c \ ../transports.c \ ../transport_common.c \ ../transport_loopback.c \ - ../transport_sctp.c \ ../transport_udp.c \ ../links_acl.c \ ../links_acl_ip.c \ ../links_acl_loopback.c \ ../lib_config.c int_timediff_test_SOURCES = int_timediff.c knet_bench_test_SOURCES = knet_bench.c \ test-common.c \ ../common.c \ ../logging.c \ ../compat.c \ ../transport_common.c \ ../threads_common.c \ ../onwire.c \ ../lib_config.c fun_pmtud_crypto_test_SOURCES = fun_pmtud_crypto.c \ test-common.c \ ../onwire.c \ ../logging.c \ ../threads_common.c \ ../lib_config.c fun_config_crypto_test_SOURCES = fun_config_crypto.c \ test-common.c fun_onwire_upgrade_test_SOURCES = fun_onwire_upgrade.c \ test-common.c fun_acl_check_test_SOURCES = fun_acl_check.c \ test-common.c diff --git a/libknet/tests/api_knet_link_set_enable.c b/libknet/tests/api_knet_link_set_enable.c index a8344c16..47e19b32 100644 --- a/libknet/tests/api_knet_link_set_enable.c +++ b/libknet/tests/api_knet_link_set_enable.c @@ -1,161 +1,89 @@ /* * Copyright (C) 2016-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * * This software licensed under GPL-2.0+ */ #include "config.h" #include #include #include #include #include #include "libknet.h" #include "internals.h" #include "link.h" #include "netutils.h" #include "test-common.h" static void test_udp(void) { knet_handle_t knet_h1, knet_h[2]; int res; int logfds[2]; struct sockaddr_storage src, dst; if (make_local_sockaddr(&src, 0) < 0) { printf("Unable to convert src to sockaddr: %s\n", strerror(errno)); exit(FAIL); } if (make_local_sockaddr(&dst, 1) < 0) { printf("Unable to convert dst to sockaddr: %s\n", strerror(errno)); exit(FAIL); } printf("Test knet_link_set_enable incorrect knet_h\n"); if ((!knet_link_set_enable(NULL, 1, 0, 1)) || (errno != EINVAL)) { printf("knet_link_set_enable accepted invalid knet_h or returned incorrect error: %s\n", strerror(errno)); exit(FAIL); } setup_logpipes(logfds); knet_h1 = knet_handle_start(logfds, KNET_LOG_DEBUG, knet_h); printf("Test knet_link_set_enable with unconfigured host_id\n"); FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, 0, 1), EINVAL); printf("Test knet_link_set_enable with incorrect linkid\n"); FAIL_ON_ERR(knet_host_add(knet_h1, 1)); FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, KNET_MAX_LINK, 1), EINVAL); printf("Test knet_link_set_enable with unconfigured link\n"); FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, 0, 1), EINVAL); printf("Test knet_link_set_enable with incorrect values\n"); FAIL_ON_ERR(knet_link_set_config(knet_h1, 1, 0, KNET_TRANSPORT_UDP, &src, &dst, 0)); FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, 0, 2), EINVAL); printf("Test knet_link_set_enable with correct values (1)\n"); FAIL_ON_ERR(knet_link_set_enable(knet_h1, 1, 0, 1)); if (knet_h1->host_index[1]->link[0].status.enabled != 1) { printf("knet_link_set_enable failed to set correct values\n"); CLEAN_EXIT(FAIL); } printf("Test knet_link_set_enable with correct values (0)\n"); FAIL_ON_ERR(knet_link_set_enable(knet_h1, 1, 0, 0)); if (knet_h1->host_index[1]->link[0].status.enabled != 0) { printf("knet_link_set_enable failed to set correct values\n"); CLEAN_EXIT(FAIL); } CLEAN_EXIT(CONTINUE); } -#ifdef HAVE_NETINET_SCTP_H -static void test_sctp(void) -{ - knet_handle_t knet_h1, knet_h[2]; - int res; - int logfds[2]; - struct sockaddr_storage src, dst; - - if (make_local_sockaddr(&src, 0) < 0) { - printf("Unable to convert src to sockaddr: %s\n", strerror(errno)); - exit(FAIL); - } - - if (make_local_sockaddr(&dst, 1) < 0) { - printf("Unable to convert dst to sockaddr: %s\n", strerror(errno)); - exit(FAIL); - } - - printf("Test knet_link_set_enable incorrect knet_h\n"); - - if ((!knet_link_set_enable(NULL, 1, 0, 1)) || (errno != EINVAL)) { - printf("knet_link_set_enable accepted invalid knet_h or returned incorrect error: %s\n", strerror(errno)); - exit(FAIL); - } - - setup_logpipes(logfds); - knet_h1 = knet_handle_start(logfds, KNET_LOG_DEBUG, knet_h); - - printf("Test knet_link_set_enable with unconfigured host_id\n"); - FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, 0, 1), EINVAL); - - printf("Test knet_link_set_enable with incorrect linkid\n"); - FAIL_ON_ERR(knet_host_add(knet_h1, 1)); - FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, KNET_MAX_LINK, 1), EINVAL); - - printf("Test knet_link_set_enable with unconfigured link\n"); - FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, 0, 1), EINVAL); - - printf("Test knet_link_set_enable with incorrect values\n"); - if (knet_link_set_config(knet_h1, 1, 0, KNET_TRANSPORT_SCTP, &src, &dst, 0) < 0) { - printf("Unable to configure link: %s\n", strerror(errno)); - CLEAN_EXIT(SKIP); - } - FAIL_ON_SUCCESS(knet_link_set_enable(knet_h1, 1, 0, 2), EINVAL); - - printf("Test knet_link_set_enable with correct values (1)\n"); - FAIL_ON_ERR(knet_link_set_enable(knet_h1, 1, 0, 1)); - if (knet_h1->host_index[1]->link[0].status.enabled != 1) { - printf("knet_link_set_enable failed to set correct values\n"); - CLEAN_EXIT(FAIL); - } - - printf("Wait 2 seconds for sockets to connect\n"); - test_sleep(knet_h1, 2); - - printf("Test knet_link_set_enable with correct values (0)\n"); - FAIL_ON_ERR(knet_link_set_enable(knet_h1, 1, 0, 0)); - if (knet_h1->host_index[1]->link[0].status.enabled != 0) { - printf("knet_link_set_enable failed to set correct values\n"); - CLEAN_EXIT(FAIL); - } - CLEAN_EXIT(CONTINUE); -} -#endif int main(int argc, char *argv[]) { printf("Testing with UDP\n"); test_udp(); -#ifdef HAVE_NETINET_SCTP_H - printf("Testing with SCTP\n"); - - test_sctp(); -#else - printf("Skipping SCTP test. Protocol not supported in this build\n"); -#endif - return PASS; } diff --git a/libknet/tests/api_knet_send.c b/libknet/tests/api_knet_send.c index 0bdcbaf3..fe151e1a 100644 --- a/libknet/tests/api_knet_send.c +++ b/libknet/tests/api_knet_send.c @@ -1,168 +1,163 @@ /* * Copyright (C) 2016-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * * This software licensed under GPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include "libknet.h" #include "internals.h" #include "netutils.h" #include "test-common.h" static int private_data; static void sock_notify(void *pvt_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno) { return; } static void test(uint8_t transport) { knet_handle_t knet_h1, knet_h[2]; int logfds[2]; int datafd = 0; int8_t channel = 0; struct knet_link_status link_status; char send_buff[KNET_MAX_PACKET_SIZE + 1]; char recv_buff[KNET_MAX_PACKET_SIZE]; ssize_t send_len = 0; int recv_len = 0; int savederrno; int res; struct sockaddr_storage lo; memset(send_buff, 0, sizeof(send_buff)); printf("Test knet_send incorrect knet_h\n"); if ((!knet_send(NULL, send_buff, KNET_MAX_PACKET_SIZE, channel)) || (errno != EINVAL)) { printf("knet_send accepted invalid knet_h or returned incorrect error: %s\n", strerror(errno)); exit(FAIL); } setup_logpipes(logfds); knet_h1 = knet_handle_start(logfds, KNET_LOG_DEBUG, knet_h); FAIL_ON_ERR(knet_handle_enable_access_lists(knet_h1, 1)); printf("Test knet_send with no send_buff\n"); FAIL_ON_SUCCESS(knet_send(knet_h1, NULL, KNET_MAX_PACKET_SIZE, channel), EINVAL); printf("Test knet_send with invalid send_buff len (0)\n"); FAIL_ON_SUCCESS(knet_send(knet_h1, send_buff, 0, channel), EINVAL); printf("Test knet_send with invalid send_buff len (> KNET_MAX_PACKET_SIZE)\n"); FAIL_ON_SUCCESS(knet_send(knet_h1, send_buff, KNET_MAX_PACKET_SIZE + 1, channel), EINVAL); printf("Test knet_send with invalid channel (-1)\n"); channel = -1; FAIL_ON_SUCCESS(knet_send(knet_h1, send_buff, KNET_MAX_PACKET_SIZE, channel), EINVAL); printf("Test knet_send with invalid channel (KNET_DATAFD_MAX)\n"); channel = KNET_DATAFD_MAX; FAIL_ON_SUCCESS(knet_send(knet_h1, send_buff, KNET_MAX_PACKET_SIZE, channel), EINVAL); printf("Test knet_send with unconfigured channel\n"); channel = 0; FAIL_ON_SUCCESS(knet_send(knet_h1, send_buff, KNET_MAX_PACKET_SIZE, channel), EINVAL); printf("Test knet_send with valid data\n"); FAIL_ON_ERR(knet_handle_enable_access_lists(knet_h1, 1)); FAIL_ON_ERR(knet_handle_enable_sock_notify(knet_h1, &private_data, sock_notify)); datafd = 0; channel = -1; FAIL_ON_ERR(knet_handle_add_datafd(knet_h1, &datafd, &channel, 0)); FAIL_ON_ERR(knet_host_add(knet_h1, 1)); if (_knet_link_set_config(knet_h1, 1, 0, transport, 0, AF_INET, 0, &lo) < 0 ) { - int exit_status = transport == KNET_TRANSPORT_SCTP && errno == EPROTONOSUPPORT ? SKIP : FAIL; + int exit_status = FAIL; printf("Unable to configure link: %s\n", strerror(errno)); CLEAN_EXIT(exit_status); } FAIL_ON_ERR(knet_link_set_enable(knet_h1, 1, 0, 1)); FAIL_ON_ERR(knet_handle_setfwd(knet_h1, 1)); FAIL_ON_ERR(wait_for_host(knet_h1, 1, 10, logfds[0], stdout)); send_len = knet_send(knet_h1, send_buff, KNET_MAX_PACKET_SIZE, channel); if (send_len <= 0) { printf("knet_send failed: %s\n", strerror(errno)); CLEAN_EXIT(FAIL); } if (send_len != sizeof(send_buff) - 1) { printf("knet_send sent only %zd bytes: %s\n", send_len, strerror(errno)); CLEAN_EXIT(FAIL); } FAIL_ON_ERR(knet_handle_setfwd(knet_h1, 0)); FAIL_ON_ERR(wait_for_packet(knet_h1, 10, datafd, logfds[0], stdout)); recv_len = knet_recv(knet_h1, recv_buff, KNET_MAX_PACKET_SIZE, channel); savederrno = errno; if (recv_len != send_len) { printf("knet_recv received only %d bytes: %s (errno: %d)\n", recv_len, strerror(errno), errno); if ((is_helgrind()) && (recv_len == -1) && (savederrno == EAGAIN)) { printf("helgrind exception. this is normal due to possible timeouts\n"); CLEAN_EXIT(PASS); } CLEAN_EXIT(FAIL); } if (memcmp(recv_buff, send_buff, KNET_MAX_PACKET_SIZE)) { printf("recv and send buffers are different!\n"); CLEAN_EXIT(FAIL); } /* A sanity check on the stats */ FAIL_ON_ERR(knet_link_get_status(knet_h1, 1, 0, &link_status, sizeof(link_status))); if (link_status.stats.tx_data_packets != 2 || link_status.stats.rx_data_packets != 2 || link_status.stats.tx_data_bytes < KNET_MAX_PACKET_SIZE || link_status.stats.rx_data_bytes < KNET_MAX_PACKET_SIZE || link_status.stats.tx_data_bytes > KNET_MAX_PACKET_SIZE*2 || link_status.stats.rx_data_bytes > KNET_MAX_PACKET_SIZE*2) { printf("stats look wrong: tx_packets: %" PRIu64 " (%" PRIu64 " bytes), rx_packets: %" PRIu64 " (%" PRIu64 " bytes)\n", link_status.stats.tx_data_packets, link_status.stats.tx_data_bytes, link_status.stats.rx_data_packets, link_status.stats.rx_data_bytes); } FAIL_ON_ERR(knet_handle_setfwd(knet_h1, 1)); CLEAN_EXIT(CONTINUE); } int main(int argc, char *argv[]) { printf("Testing with UDP\n"); test(KNET_TRANSPORT_UDP); -#ifdef HAVE_NETINET_SCTP_H - printf("Testing with SCTP\n"); - test(KNET_TRANSPORT_SCTP); -#endif - return PASS; } diff --git a/libknet/tests/fun_acl_check.c b/libknet/tests/fun_acl_check.c index 4d444361..c9f4f64e 100644 --- a/libknet/tests/fun_acl_check.c +++ b/libknet/tests/fun_acl_check.c @@ -1,404 +1,395 @@ /* * Copyright (C) 2021-2025 Red Hat, Inc. All rights reserved. * * Authors: Christine Caulfield * * This software licensed under GPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include #include #include "libknet.h" #include "internals.h" #include "netutils.h" #include "test-common.h" /* * Keep track of how many messages got through: * clean + 3xACLs + QUIT */ #define CORRECT_NUM_MSGS 5 static int msgs_recvd = 0; #undef TESTNODES #define TESTNODES 2 static pthread_mutex_t recv_mutex = PTHREAD_MUTEX_INITIALIZER; static int quit_recv_thread = 0; static int reply_pipe[2]; /* Our local version of FOE that also tidies up the threads */ #define FAIL_ON_ERR_THR(fn) \ printf("FOE: %s\n", #fn); \ if ((res = fn) != 0) { \ int savederrno = errno; \ pthread_mutex_lock(&recv_mutex); \ quit_recv_thread = 1; \ pthread_mutex_unlock(&recv_mutex); \ if (recv_thread) { \ pthread_join(recv_thread, (void**)&thread_err); \ } \ knet_handle_stop_everything(knet_h, TESTNODES); \ stop_logthread(); \ flush_logs(logfds[0], stdout); \ close_logpipes(logfds); \ close(reply_pipe[0]); \ close(reply_pipe[1]); \ if (res == -2) { \ exit(SKIP); \ } else { \ printf("*** FAIL on line %d %s failed: %s\n", __LINE__ , #fn, strerror(savederrno)); \ exit(FAIL); \ } \ } static int knet_send_str(knet_handle_t knet_h, char *str) { return knet_send_sync(knet_h, str, strlen(str)+1, 0); } /* * lo0 is filled in with the local address on return. * lo1 is expected to be provided - it's the actual remote address to connect to. */ int dyn_knet_link_set_config(knet_handle_t knet_h, knet_node_id_t host_id, uint8_t link_id, uint8_t transport, uint64_t flags, int family, int dynamic, struct sockaddr_storage *lo0, struct sockaddr_storage *lo1) { int err = 0, savederrno = 0; uint32_t port; char portstr[32]; for (port = 1025; port < 65536; port++) { sprintf(portstr, "%u", port); memset(lo0, 0, sizeof(struct sockaddr_storage)); if (family == AF_INET6) { err = knet_strtoaddr("::1", portstr, lo0, sizeof(struct sockaddr_storage)); } else { err = knet_strtoaddr("127.0.0.1", portstr, lo0, sizeof(struct sockaddr_storage)); } if (err < 0) { printf("Unable to convert loopback to sockaddr: %s\n", strerror(errno)); goto out; } errno = 0; if (dynamic) { err = knet_link_set_config(knet_h, host_id, link_id, transport, lo0, NULL, flags); } else { err = knet_link_set_config(knet_h, host_id, link_id, transport, lo0, lo1, flags); } savederrno = errno; if ((err < 0) && (savederrno != EADDRINUSE)) { - if (savederrno == EPROTONOSUPPORT && transport == KNET_TRANSPORT_SCTP) { - return -2; - } else { - printf("Unable to configure link: %s\n", strerror(savederrno)); - goto out; - } + printf("Unable to configure link: %s\n", strerror(savederrno)); + goto out; } if (!err) { printf("Using port %u\n", port); goto out; } } if (err) { printf("No more ports available\n"); } out: errno = savederrno; return err; } static void *recv_messages(void *handle) { knet_handle_t knet_h = (knet_handle_t)handle; char buf[4096]; ssize_t len; static int err = 0; int savederrno = 0, quit = 0; while ((len = knet_recv(knet_h, buf, sizeof(buf), 0)) && (!quit)) { savederrno = errno; pthread_mutex_lock(&recv_mutex); quit = quit_recv_thread; pthread_mutex_unlock(&recv_mutex); if (quit) { printf(" *** recv thread was requested to exit via FOE\n"); err = 1; return &err; } if (len > 0) { int res; printf("recv: (%ld) %s\n", (long)len, buf); msgs_recvd++; if (strcmp("QUIT", buf) == 0) { break; } if (buf[0] == '0') { /* We should not have received this! */ printf(" *** FAIL received packet that should have been blocked\n"); err = 1; return &err; } /* Tell the main thread we have received something */ res = write(reply_pipe[1], ".", 1); if (res != 1) { printf(" *** FAIL to send response back to main thread\n"); err = 1; return &err; } } usleep(1000); if (len < 0 && savederrno != EAGAIN) { break; } } printf("-- recv thread finished: %zd %d %s\n", len, errno, strerror(savederrno)); return &err; } static void notify_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno) { printf("NOTIFY fn called\n"); } /* A VERY basic filter because all data traffic is going to one place */ static int dhost_filter(void *pvt_data, const unsigned char *outdata, ssize_t outdata_len, uint8_t tx_rx, knet_node_id_t this_host_id, knet_node_id_t src_host_id, int8_t *dst_channel, knet_node_id_t *dst_host_ids, size_t *dst_host_ids_entries) { dst_host_ids[0] = 1; *dst_host_ids_entries = 1; return 0; } /* This used to be a pthread condition variable, but there was a race where it could be triggered before the main thread was waiting for it. Go old-fashioned. */ static int wait_for_reply(int seconds) { int res; struct pollfd pfds; char tmpbuf[32]; pfds.fd = reply_pipe[0]; pfds.events = POLLIN | POLLERR | POLLHUP; pfds.revents = 0; res = poll(&pfds, 1, seconds*1000); if (res == 1) { if (pfds.revents & POLLIN) { res = read(reply_pipe[0], tmpbuf, sizeof(tmpbuf)); if (res > 0) { return 0; } } else { printf("Error on pipe poll revent = 0x%x\n", pfds.revents); errno = EIO; } } if (res == 0) { errno = ETIMEDOUT; return -1; } return -1; } static void test(int transport) { knet_handle_t knet_h[TESTNODES+1]; int logfds[2]; struct sockaddr_storage lo0, lo1; struct sockaddr_storage ss1, ss2; int res; pthread_t recv_thread = 0; int *thread_err; int datafd; int8_t channel; int seconds = 90; // dynamic tests take longer than normal tests if (is_memcheck() || is_helgrind()) { printf("Test suite is running under valgrind, adjusting wait_for_host timeout\n"); seconds = seconds * 16; } memset(knet_h, 0, sizeof(knet_h)); memset(reply_pipe, 0, sizeof(reply_pipe)); memset(logfds, 0, sizeof(logfds)); FAIL_ON_ERR_THR(pipe(reply_pipe)); // Initial setup gubbins msgs_recvd = 0; setup_logpipes(logfds); start_logthread(logfds[1], stdout); knet_handle_start_nodes(knet_h, TESTNODES, logfds, KNET_LOG_DEBUG); FAIL_ON_ERR_THR(knet_host_add(knet_h[2], 1)); FAIL_ON_ERR_THR(knet_host_add(knet_h[1], 2)); FAIL_ON_ERR_THR(knet_handle_enable_filter(knet_h[2], NULL, dhost_filter)); // Create the dynamic (receiving) link FAIL_ON_ERR_THR(dyn_knet_link_set_config(knet_h[1], 2, 0, transport, 0, AF_INET, 1, &lo0, NULL)); // Connect to the dynamic link FAIL_ON_ERR_THR(dyn_knet_link_set_config(knet_h[2], 1, 0, transport, 0, AF_INET, 0, &lo1, &lo0)); // All the rest of the setup gubbins FAIL_ON_ERR_THR(knet_handle_enable_sock_notify(knet_h[1], 0, ¬ify_fn)); FAIL_ON_ERR_THR(knet_handle_enable_sock_notify(knet_h[2], 0, ¬ify_fn)); channel = datafd = 0; FAIL_ON_ERR_THR(knet_handle_add_datafd(knet_h[1], &datafd, &channel, 0)); channel = datafd = 0; FAIL_ON_ERR_THR(knet_handle_add_datafd(knet_h[2], &datafd, &channel, 0)); FAIL_ON_ERR_THR(knet_link_set_enable(knet_h[1], 2, 0, 1)); FAIL_ON_ERR_THR(knet_link_set_enable(knet_h[2], 1, 0, 1)); FAIL_ON_ERR_THR(knet_handle_setfwd(knet_h[1], 1)); FAIL_ON_ERR_THR(knet_handle_setfwd(knet_h[2], 1)); // Start receive thread FAIL_ON_ERR_THR(pthread_create(&recv_thread, NULL, recv_messages, (void *)knet_h[1])); // Let everything settle down FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 1, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 1, seconds, logfds[0], stdout)); /* * TESTING STARTS HERE * strings starting '1' should reach the receiving thread * strings starting '0' should not */ // No ACL printf("Testing No ACL - this should get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "1No ACL - this should get through")); FAIL_ON_ERR_THR(wait_for_reply(seconds)) // Block traffic from this address. memset(&ss1, 0, sizeof(ss1)); memset(&ss2, 0, sizeof(ss1)); knet_strtoaddr("127.0.0.1","0", &ss1, sizeof(ss1)); FAIL_ON_ERR_THR(knet_link_add_acl(knet_h[1], 2, 0, &ss1, NULL, CHECK_TYPE_ADDRESS, CHECK_REJECT)); // Accept ACL for when we remove them FAIL_ON_ERR_THR(knet_link_add_acl(knet_h[1], 2, 0, &ss1, NULL, CHECK_TYPE_ADDRESS, CHECK_ACCEPT)); // This needs to go after the first ACLs are added FAIL_ON_ERR_THR(knet_handle_enable_access_lists(knet_h[1], 1)); printf("Testing Address blocked - this should NOT get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "0Address blocked - this should NOT get through")); // Unblock and check again FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 0, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 0, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(knet_link_rm_acl(knet_h[1], 2, 0, &ss1, NULL, CHECK_TYPE_ADDRESS, CHECK_REJECT)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 1, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 1, seconds, logfds[0], stdout)); printf("Testing Address unblocked - this should get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "1Address unblocked - this should get through")); FAIL_ON_ERR_THR(wait_for_reply(seconds)); // Block traffic using a netmask knet_strtoaddr("127.0.0.1","0", &ss1, sizeof(ss1)); knet_strtoaddr("255.0.0.1","0", &ss2, sizeof(ss2)); FAIL_ON_ERR_THR(knet_link_insert_acl(knet_h[1], 2, 0, 0, &ss1, &ss2, CHECK_TYPE_MASK, CHECK_REJECT)); printf("Testing Netmask blocked - this should NOT get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "0Netmask blocked - this should NOT get through")); // Unblock and check again FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 0, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 0, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(knet_link_rm_acl(knet_h[1], 2, 0, &ss1, &ss2, CHECK_TYPE_MASK, CHECK_REJECT)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 1, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 1, seconds, logfds[0], stdout)); printf("Testing Netmask unblocked - this should get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "1Netmask unblocked - this should get through")); FAIL_ON_ERR_THR(wait_for_reply(seconds)); // Block traffic from a range knet_strtoaddr("127.0.0.0", "0", &ss1, sizeof(ss1)); knet_strtoaddr("127.0.0.9", "0", &ss2, sizeof(ss2)); FAIL_ON_ERR_THR(knet_link_insert_acl(knet_h[1], 2, 0, 0, &ss1, &ss2, CHECK_TYPE_RANGE, CHECK_REJECT)); printf("Testing Range blocked - this should NOT get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "0Range blocked - this should NOT get through")); // Unblock and check again FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 0, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 0, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(knet_link_rm_acl(knet_h[1], 2, 0, &ss1, &ss2, CHECK_TYPE_RANGE, CHECK_REJECT)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 1, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 1, seconds, logfds[0], stdout)); printf("Testing Range unblocked - this should get through\n"); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "1Range unblocked - this should get through")); FAIL_ON_ERR_THR(wait_for_reply(seconds)); // Finish up - disable ACLS to make sure the QUIT message gets through FAIL_ON_ERR_THR(knet_handle_enable_access_lists(knet_h[1], 0)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[1], TESTNODES, 1, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(wait_for_nodes_state(knet_h[2], TESTNODES, 1, seconds, logfds[0], stdout)); FAIL_ON_ERR_THR(knet_send_str(knet_h[2], "QUIT")); // Check return from the receiving thread pthread_join(recv_thread, (void**)&thread_err); if (*thread_err) { printf("Thread returned %d\n", *thread_err); clean_exit(knet_h, TESTNODES, logfds, FAIL); } if (msgs_recvd != CORRECT_NUM_MSGS) { printf("*** FAIL Recv thread got %d messages, expected %d\n", msgs_recvd, CORRECT_NUM_MSGS); clean_exit(knet_h, TESTNODES, logfds, FAIL); } clean_exit(knet_h, TESTNODES, logfds, PASS); } int main(int argc, char *argv[]) { printf("Testing with UDP\n"); test(KNET_TRANSPORT_UDP); -#ifdef HAVE_NETINET_SCTP_H - printf("Testing with SCTP currently disabled\n"); - //test(KNET_TRANSPORT_SCTP); -#endif - return PASS; } diff --git a/libknet/tests/knet_bench.c b/libknet/tests/knet_bench.c index e31875fe..1552ffd7 100644 --- a/libknet/tests/knet_bench.c +++ b/libknet/tests/knet_bench.c @@ -1,1379 +1,1375 @@ /* * Copyright (C) 2016-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * * This software licensed under GPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include #include #include #include "libknet.h" #include "compat.h" #include "internals.h" #include "netutils.h" #include "transport_common.h" #include "test-common.h" #define MAX_NODES 128 static int senderid = -1; static int thisnodeid = -1; static knet_handle_t knet_h; static int datafd = 0; static int8_t channel = 0; static int globallistener = 0; static int continous = 0; static int show_stats = 0; static struct sockaddr_storage allv4; static struct sockaddr_storage allv6; static int broadcast_test = 1; static pthread_t rx_thread = {0}; static char *rx_buf[PCKT_FRAG_MAX]; static int wait_for_perf_rx = 0; static char *compresscfg = NULL; static char *cryptocfg = NULL; static int machine_output = 0; static int use_access_lists = 0; static int use_pckt_verification = 0; static int bench_shutdown_in_progress = 0; static pthread_mutex_t shutdown_mutex = PTHREAD_MUTEX_INITIALIZER; #define TEST_PING 0 #define TEST_PING_AND_DATA 1 #define TEST_PERF_BY_SIZE 2 #define TEST_PERF_BY_TIME 3 static int test_type = TEST_PING; #define TEST_START 2 #define TEST_STOP 4 #define TEST_COMPLETE 6 #define ONE_GIGABYTE 1073741824 static uint64_t perf_by_size_size = 1 * ONE_GIGABYTE; static uint64_t perf_by_time_secs = 10; static uint32_t force_packet_size = 0; struct node { int nodeid; int links; uint8_t transport[KNET_MAX_LINK]; struct sockaddr_storage address[KNET_MAX_LINK]; }; struct pckt_ver { uint32_t len; uint32_t chksum; }; static void print_help(void) { printf("knet_bench usage:\n"); printf(" -h print this help (no really)\n"); printf(" -d enable debug logs (default INFO)\n"); printf(" -f enable use of access lists (default: off)\n"); printf(" -c [implementation]:[crypto]:[hashing] crypto configuration. (default disabled)\n"); printf(" Example: -c nss:aes128:sha1\n"); printf(" -z [implementation]:[level]:[threshold] compress configuration. (default disabled)\n"); printf(" Example: -z zlib:5:100\n"); printf(" -p [active|passive|rr] (default: passive)\n"); - printf(" -P [UDP|SCTP] (default: UDP) protocol (transport) to use for all links\n"); + printf(" -P [UDP] (default: UDP) protocol (transport) to use for all links, only UDP supported\n"); printf(" -t [nodeid] This nodeid (required)\n"); printf(" -n [nodeid],[proto]/[link1_ip],[link2_..] Other nodes information (at least one required)\n"); - printf(" Example: -n 1,192.168.8.1,SCTP/3ffe::8:1,UDP/172...\n"); + printf(" Example: -n 1,192.168.8.1,UDP/3ffe::8:1,UDP/172...\n"); printf(" can be repeated up to %d and should contain also the localnode info\n", MAX_NODES); printf(" -b [port] baseport (default: 50000)\n"); printf(" -l enable global listener on 0.0.0.0/:: (default: off, incompatible with -o)\n"); printf(" -o enable baseport offset per nodeid\n"); printf(" -m change PMTUd interval in seconds (default: 60)\n"); printf(" -w dont wait for all nodes to be up before starting the test (default: wait)\n"); printf(" -T [ping|ping_data|perf-by-size|perf-by-time]\n"); printf(" test type (default: ping)\n"); printf(" ping: will wait for all hosts to join the knet network, sleep 5 seconds and quit\n"); printf(" ping_data: will wait for all hosts to join the knet network, sends some data to all nodes and quit\n"); printf(" perf-by-size: will wait for all hosts to join the knet network,\n"); printf(" perform a series of benchmarks by transmitting a known\n"); printf(" size/quantity of packets and measuring the time, then quit\n"); printf(" perf-by-time: will wait for all hosts to join the knet network,\n"); printf(" perform a series of benchmarks by transmitting a known\n"); printf(" size of packets for a given amount of time (10 seconds)\n"); printf(" and measuring the quantity of data transmitted, then quit\n"); printf(" -s nodeid that will generate traffic for benchmarks\n"); printf(" -S [size|seconds] when used in combination with -T perf-by-size it indicates how many GB of traffic to generate for the test. (default: 1GB)\n"); printf(" when used in combination with -T perf-by-time it indicates how many Seconds of traffic to generate for the test. (default: 10 seconds)\n"); printf(" -x force packet size for perf-by-time or perf-by-size\n"); printf(" -C repeat the test continously (default: off)\n"); printf(" -X[XX] show stats at the end of the run (default: 1)\n"); printf(" 1: show handle stats, 2: show summary link stats\n"); printf(" 3: show detailed link stats\n"); printf(" -a enable machine parsable output (default: off).\n"); printf(" -v enable packet verification for performance tests (default: off).\n"); } static void parse_nodes(char *nodesinfo[MAX_NODES], int onidx, int port, struct node nodes[MAX_NODES], int *thisidx) { int i; char *temp = NULL; char port_str[11]; memset(port_str, 0, sizeof(port_str)); snprintf(port_str, sizeof(port_str), "%d", port); for (i = 0; i < onidx; i++) { nodes[i].nodeid = atoi(strtok(nodesinfo[i], ",")); if ((nodes[i].nodeid < 0) || (nodes[i].nodeid > KNET_MAX_HOST)) { printf("Invalid nodeid: %d (0 - %d)\n", nodes[i].nodeid, KNET_MAX_HOST); exit(FAIL); } if (thisnodeid == nodes[i].nodeid) { *thisidx = i; } while((temp = strtok(NULL, ","))) { char *slash = NULL; uint8_t transport; if (nodes[i].links == KNET_MAX_LINK) { printf("Too many links configured. Max %d\n", KNET_MAX_LINK); exit(FAIL); } slash = strstr(temp, "/"); if (slash) { memset(slash, 0, 1); transport = knet_get_transport_id_by_name(temp); if (transport == KNET_MAX_TRANSPORTS) { printf("Unknown transport: %s\n", temp); exit(FAIL); } nodes[i].transport[nodes[i].links] = transport; temp = slash + 1; } else { nodes[i].transport[nodes[i].links] = KNET_TRANSPORT_UDP; } if (knet_strtoaddr(temp, port_str, &nodes[i].address[nodes[i].links], sizeof(struct sockaddr_storage)) < 0) { printf("Unable to convert %s to sockaddress\n", temp); exit(FAIL); } nodes[i].links++; } } if (knet_strtoaddr("0.0.0.0", port_str, &allv4, sizeof(struct sockaddr_storage)) < 0) { printf("Unable to convert 0.0.0.0 to sockaddress\n"); exit(FAIL); } if (knet_strtoaddr("::", port_str, &allv6, sizeof(struct sockaddr_storage)) < 0) { printf("Unable to convert :: to sockaddress\n"); exit(FAIL); } for (i = 1; i < onidx; i++) { if (nodes[0].links != nodes[i].links) { printf("knet_bench does not support unbalanced link configuration\n"); exit(FAIL); } } return; } static int private_data; static void sock_notify(void *pvt_data, int local_datafd, int8_t local_channel, uint8_t tx_rx, int error, int errorno) { printf("[info]: error (%d - %d - %s) from socket: %d\n", error, errorno, strerror(errno), local_datafd); return; } static int ping_dst_host_filter(void *pvt_data, const unsigned char *outdata, ssize_t outdata_len, uint8_t tx_rx, knet_node_id_t this_host_id, knet_node_id_t src_host_id, int8_t *dst_channel, knet_node_id_t *dst_host_ids, size_t *dst_host_ids_entries) { if (broadcast_test) { return 1; } if (tx_rx == KNET_NOTIFY_TX) { memmove(&dst_host_ids[0], outdata, 2); } else { dst_host_ids[0] = this_host_id; } *dst_host_ids_entries = 1; return 0; } static void setup_knet(int argc, char *argv[]) { int logfd = 0; int rv; char *policystr = NULL, *protostr = NULL; char *othernodeinfo[MAX_NODES]; struct node nodes[MAX_NODES]; int thisidx = -1; int onidx = 0; int debug = KNET_LOG_INFO; int port = 50000, portoffset = 0; int thisport = 0, otherport = 0; int thisnewport = 0, othernewport = 0; struct sockaddr_in *so_in; struct sockaddr_in6 *so_in6; struct sockaddr_storage *src; int i, link_idx, allnodesup = 0; int policy = KNET_LINK_POLICY_PASSIVE, policyfound = 0; int protocol = KNET_TRANSPORT_UDP, protofound = 0; int wait = 1; int pmtud_interval = 60; struct knet_handle_crypto_cfg knet_handle_crypto_cfg; char *cryptomodel = NULL, *cryptotype = NULL, *cryptohash = NULL; struct knet_handle_compress_cfg knet_handle_compress_cfg; memset(nodes, 0, sizeof(nodes)); while ((rv = getopt(argc, argv, "aCT:S:s:lvdfom:wb:t:n:c:p:x:X::P:z:h")) != EOF) { switch(rv) { case 'h': print_help(); exit(PASS); break; case 'a': machine_output = 1; break; case 'd': debug = KNET_LOG_DEBUG; break; case 'f': use_access_lists = 1; break; case 'c': if (cryptocfg) { printf("Error: -c can only be specified once\n"); exit(FAIL); } cryptocfg = optarg; break; case 'p': if (policystr) { printf("Error: -p can only be specified once\n"); exit(FAIL); } if (optarg) { policystr = optarg; if (!strcmp(policystr, "active")) { policy = KNET_LINK_POLICY_ACTIVE; policyfound = 1; } /* * we can't use rr because clangs can't compile * an array of 3 strings, one of which is 2 bytes long */ if (!strcmp(policystr, "round-robin")) { policy = KNET_LINK_POLICY_RR; policyfound = 1; } if (!strcmp(policystr, "passive")) { policy = KNET_LINK_POLICY_PASSIVE; policyfound = 1; } } if (!policyfound) { printf("Error: invalid policy %s specified. -p accepts active|passive|rr\n", policystr); exit(FAIL); } break; case 'P': if (protostr) { printf("Error: -P can only be specified once\n"); exit(FAIL); } if (optarg) { protostr = optarg; if (!strcmp(protostr, "UDP")) { protocol = KNET_TRANSPORT_UDP; protofound = 1; } - if (!strcmp(protostr, "SCTP")) { - protocol = KNET_TRANSPORT_SCTP; - protofound = 1; - } } if (!protofound) { - printf("Error: invalid protocol %s specified. -P accepts udp|sctp\n", policystr); + printf("Error: invalid protocol %s specified. -P accepts udp\n", policystr); exit(FAIL); } break; case 't': if (thisnodeid >= 0) { printf("Error: -t can only be specified once\n"); exit(FAIL); } thisnodeid = atoi(optarg); if ((thisnodeid < 0) || (thisnodeid > 65536)) { printf("Error: -t nodeid out of range %d (1 - 65536)\n", thisnodeid); exit(FAIL); } break; case 'n': if (onidx == MAX_NODES) { printf("Error: too many other nodes. Max %d\n", MAX_NODES); exit(FAIL); } othernodeinfo[onidx] = optarg; onidx++; break; case 'b': port = atoi(optarg); if ((port < 1) || (port > 65536)) { printf("Error: port %d out of range (1 - 65536)\n", port); exit(FAIL); } break; case 'o': if (globallistener) { printf("Error: -l cannot be used with -o\n"); exit(FAIL); } portoffset = 1; break; case 'm': pmtud_interval = atoi(optarg); if (pmtud_interval < 1) { printf("Error: pmtud interval %d out of range (> 0)\n", pmtud_interval); exit(FAIL); } break; case 'l': if (portoffset) { printf("Error: -o cannot be used with -l\n"); exit(FAIL); } globallistener = 1; break; case 'w': wait = 0; break; case 's': if (senderid >= 0) { printf("Error: -s can only be specified once\n"); exit(FAIL); } senderid = atoi(optarg); if ((senderid < 0) || (senderid > 65536)) { printf("Error: -s nodeid out of range %d (1 - 65536)\n", senderid); exit(FAIL); } break; case 'T': if (optarg) { if (!strcmp("ping", optarg)) { test_type = TEST_PING; } if (!strcmp("ping_data", optarg)) { test_type = TEST_PING_AND_DATA; } if (!strcmp("perf-by-size", optarg)) { test_type = TEST_PERF_BY_SIZE; } if (!strcmp("perf-by-time", optarg)) { test_type = TEST_PERF_BY_TIME; } } else { printf("Error: -T requires an option\n"); exit(FAIL); } break; case 'S': perf_by_size_size = (uint64_t)atoi(optarg) * ONE_GIGABYTE; perf_by_time_secs = (uint64_t)atoi(optarg); break; case 'x': force_packet_size = (uint32_t)atoi(optarg); if ((force_packet_size < 64) || (force_packet_size > 65536)) { printf("Unsupported packet size %u (accepted 64 - 65536)\n", force_packet_size); exit(FAIL); } break; case 'v': use_pckt_verification = 1; break; case 'C': continous = 1; break; case 'X': if (optarg) { show_stats = atoi(optarg); } else { show_stats = 1; } break; case 'z': if (compresscfg) { printf("Error: -c can only be specified once\n"); exit(FAIL); } compresscfg = optarg; break; default: break; } } if (thisnodeid < 0) { printf("Who am I?!? missing -t from command line?\n"); exit(FAIL); } if (onidx < 1) { printf("no other nodes configured?!? missing -n from command line\n"); exit(FAIL); } parse_nodes(othernodeinfo, onidx, port, nodes, &thisidx); if (thisidx < 0) { printf("no config for this node found\n"); exit(FAIL); } if (senderid >= 0) { for (i=0; i < onidx; i++) { if (senderid == nodes[i].nodeid) { break; } } if (i == onidx) { printf("Unable to find senderid in nodelist\n"); exit(FAIL); } } if (((test_type == TEST_PERF_BY_SIZE) || (test_type == TEST_PERF_BY_TIME)) && (senderid < 0)) { printf("Error: performance test requires -s to be set (for now)\n"); exit(FAIL); } logfd = start_logging(stdout); knet_h = knet_handle_new(thisnodeid, logfd, debug, 0); if (!knet_h) { printf("Unable to knet_handle_new: %s\n", strerror(errno)); exit(FAIL); } if (knet_handle_enable_access_lists(knet_h, use_access_lists) < 0) { printf("Unable to knet_handle_enable_access_lists: %s\n", strerror(errno)); exit(FAIL); } if (knet_handle_set_host_defrag_bufs(knet_h, 4, KNET_MAX_DEFRAG_BUFS_DEFAULT, KNET_SHRINK_THRESHOLD_DEFAULT, RECLAIM_POLICY_ABSOLUTE) < 0) { printf("Unable to knet_handle_set_host_defrag_bufs: %s\n", strerror(errno)); exit(FAIL); } if (cryptocfg) { memset(&knet_handle_crypto_cfg, 0, sizeof(knet_handle_crypto_cfg)); cryptomodel = strtok(cryptocfg, ":"); cryptotype = strtok(NULL, ":"); cryptohash = strtok(NULL, ":"); if (cryptomodel) { strncpy(knet_handle_crypto_cfg.crypto_model, cryptomodel, sizeof(knet_handle_crypto_cfg.crypto_model) - 1); } if (cryptotype) { strncpy(knet_handle_crypto_cfg.crypto_cipher_type, cryptotype, sizeof(knet_handle_crypto_cfg.crypto_cipher_type) - 1); } if (cryptohash) { strncpy(knet_handle_crypto_cfg.crypto_hash_type, cryptohash, sizeof(knet_handle_crypto_cfg.crypto_hash_type) - 1); } knet_handle_crypto_cfg.private_key_len = KNET_MAX_KEY_LEN; if (knet_handle_crypto_set_config(knet_h, &knet_handle_crypto_cfg, 1)) { printf("Unable to set crypto config\n"); exit(FAIL); } if (knet_handle_crypto_use_config(knet_h, 1)) { printf("Unable to use crypto config\n"); exit(FAIL); } if (knet_handle_crypto_rx_clear_traffic(knet_h, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC)) { printf("Unable to disable clear traffic on RX\n"); exit(FAIL); } } if (compresscfg) { memset(&knet_handle_compress_cfg, 0, sizeof(struct knet_handle_compress_cfg)); snprintf(knet_handle_compress_cfg.compress_model, 16, "%s", strtok(compresscfg, ":")); knet_handle_compress_cfg.compress_level = atoi(strtok(NULL, ":")); knet_handle_compress_cfg.compress_threshold = atoi(strtok(NULL, ":")); if (knet_handle_compress(knet_h, &knet_handle_compress_cfg)) { printf("Unable to configure compress\n"); exit(FAIL); } } if (knet_handle_enable_sock_notify(knet_h, &private_data, sock_notify) < 0) { printf("knet_handle_enable_sock_notify failed: %s\n", strerror(errno)); knet_handle_free(knet_h); exit(FAIL); } datafd = 0; channel = -1; if (knet_handle_add_datafd(knet_h, &datafd, &channel, 0) < 0) { printf("knet_handle_add_datafd failed: %s\n", strerror(errno)); knet_handle_free(knet_h); exit(FAIL); } if (knet_handle_pmtud_setfreq(knet_h, pmtud_interval) < 0) { printf("knet_handle_pmtud_setfreq failed: %s\n", strerror(errno)); knet_handle_free(knet_h); exit(FAIL); } for (i=0; i < onidx; i++) { if (i == thisidx) { continue; } if (knet_host_add(knet_h, nodes[i].nodeid) < 0) { printf("knet_host_add failed: %s\n", strerror(errno)); exit(FAIL); } if (knet_host_set_policy(knet_h, nodes[i].nodeid, policy) < 0) { printf("knet_host_set_policy failed: %s\n", strerror(errno)); exit(FAIL); } for (link_idx = 0; link_idx < nodes[i].links; link_idx++) { if (portoffset) { if (nodes[thisidx].address[link_idx].ss_family == AF_INET) { so_in = (struct sockaddr_in *)&nodes[thisidx].address[link_idx]; thisport = ntohs(so_in->sin_port); thisnewport = thisport + nodes[i].nodeid; so_in->sin_port = (htons(thisnewport)); so_in = (struct sockaddr_in *)&nodes[i].address[link_idx]; otherport = ntohs(so_in->sin_port); othernewport = otherport + nodes[thisidx].nodeid; so_in->sin_port = (htons(othernewport)); } else { so_in6 = (struct sockaddr_in6 *)&nodes[thisidx].address[link_idx]; thisport = ntohs(so_in6->sin6_port); thisnewport = thisport + nodes[i].nodeid; so_in6->sin6_port = (htons(thisnewport)); so_in6 = (struct sockaddr_in6 *)&nodes[i].address[link_idx]; otherport = ntohs(so_in6->sin6_port); othernewport = otherport + nodes[thisidx].nodeid; so_in6->sin6_port = (htons(othernewport)); } } if (!globallistener) { src = &nodes[thisidx].address[link_idx]; } else { if (nodes[thisidx].address[link_idx].ss_family == AF_INET) { src = &allv4; } else { src = &allv6; } } /* * -P overrides per link protocol configuration */ if (protofound) { nodes[i].transport[link_idx] = protocol; } if (knet_link_set_config(knet_h, nodes[i].nodeid, link_idx, nodes[i].transport[link_idx], src, &nodes[i].address[link_idx], 0) < 0) { printf("Unable to configure link: %s\n", strerror(errno)); exit(FAIL); } if (portoffset) { if (nodes[thisidx].address[link_idx].ss_family == AF_INET) { so_in = (struct sockaddr_in *)&nodes[thisidx].address[link_idx]; so_in->sin_port = (htons(thisport)); so_in = (struct sockaddr_in *)&nodes[i].address[link_idx]; so_in->sin_port = (htons(otherport)); } else { so_in6 = (struct sockaddr_in6 *)&nodes[thisidx].address[link_idx]; so_in6->sin6_port = (htons(thisport)); so_in6 = (struct sockaddr_in6 *)&nodes[i].address[link_idx]; so_in6->sin6_port = (htons(otherport)); } } if (knet_link_set_enable(knet_h, nodes[i].nodeid, link_idx, 1) < 0) { printf("knet_link_set_enable failed: %s\n", strerror(errno)); exit(FAIL); } if (knet_link_set_ping_timers(knet_h, nodes[i].nodeid, link_idx, 1000, 10000, 2048) < 0) { printf("knet_link_set_ping_timers failed: %s\n", strerror(errno)); exit(FAIL); } if (knet_link_set_pong_count(knet_h, nodes[i].nodeid, link_idx, 2) < 0) { printf("knet_link_set_pong_count failed: %s\n", strerror(errno)); exit(FAIL); } } } if (knet_handle_enable_filter(knet_h, NULL, ping_dst_host_filter)) { printf("Unable to enable dst_host_filter: %s\n", strerror(errno)); exit(FAIL); } if (knet_handle_setfwd(knet_h, 1) < 0) { printf("knet_handle_setfwd failed: %s\n", strerror(errno)); exit(FAIL); } if (wait) { while(!allnodesup) { allnodesup = 1; for (i=0; i < onidx; i++) { if (i == thisidx) { continue; } if (knet_h->host_index[nodes[i].nodeid]->status.reachable != 1) { printf("[info]: waiting host %d to be reachable\n", nodes[i].nodeid); allnodesup = 0; } } if (!allnodesup) { sleep(1); } } sleep(1); } } static void *_rx_thread(void *args) { int rx_epoll; struct epoll_event ev; struct epoll_event events[KNET_EPOLL_MAX_EVENTS]; struct sockaddr_storage address[PCKT_FRAG_MAX]; struct knet_mmsghdr msg[PCKT_FRAG_MAX]; struct iovec iov_in[PCKT_FRAG_MAX]; int i, msg_recv; struct timespec clock_start, clock_end; unsigned long long time_diff = 0; uint64_t rx_pkts = 0; uint64_t rx_bytes = 0; unsigned int current_pckt_size = 0; for (i = 0; (unsigned int)i < PCKT_FRAG_MAX; i++) { rx_buf[i] = malloc(KNET_MAX_PACKET_SIZE); if (!rx_buf[i]) { printf("RXT: Unable to malloc!\nHALTING RX THREAD!\n"); return NULL; } memset(rx_buf[i], 0, KNET_MAX_PACKET_SIZE); iov_in[i].iov_base = (void *)rx_buf[i]; iov_in[i].iov_len = KNET_MAX_PACKET_SIZE; memset(&msg[i].msg_hdr, 0, sizeof(struct msghdr)); msg[i].msg_hdr.msg_name = &address[i]; msg[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_storage); msg[i].msg_hdr.msg_iov = &iov_in[i]; msg[i].msg_hdr.msg_iovlen = 1; } rx_epoll = epoll_create(KNET_EPOLL_MAX_EVENTS + 1); if (rx_epoll < 0) { printf("RXT: Unable to create epoll!\nHALTING RX THREAD!\n"); return NULL; } memset(&ev, 0, sizeof(struct epoll_event)); ev.events = EPOLLIN; ev.data.fd = datafd; if (epoll_ctl(rx_epoll, EPOLL_CTL_ADD, datafd, &ev)) { printf("RXT: Unable to add datafd to epoll\nHALTING RX THREAD!\n"); return NULL; } memset(&clock_start, 0, sizeof(clock_start)); memset(&clock_end, 0, sizeof(clock_start)); while (!bench_shutdown_in_progress) { if (epoll_wait(rx_epoll, events, KNET_EPOLL_MAX_EVENTS, 1) >= 1) { msg_recv = _recvmmsg(datafd, &msg[0], PCKT_FRAG_MAX, MSG_DONTWAIT | MSG_NOSIGNAL); if (msg_recv < 0) { printf("[info]: RXT: error from recvmmsg: %s\n", strerror(errno)); } switch(test_type) { case TEST_PING_AND_DATA: for (i = 0; i < msg_recv; i++) { if (msg[i].msg_len == 0) { printf("[info]: RXT: received 0 bytes message?\n"); } printf("[info]: received %u bytes message: %s\n", msg[i].msg_len, (char *)msg[i].msg_hdr.msg_iov->iov_base); } break; case TEST_PERF_BY_TIME: case TEST_PERF_BY_SIZE: for (i = 0; i < msg_recv; i++) { if (msg[i].msg_len < 64) { if (msg[i].msg_len == 0) { printf("[info]: RXT: received 0 bytes message?\n"); } if (msg[i].msg_len == TEST_START) { if (clock_gettime(CLOCK_MONOTONIC, &clock_start) != 0) { printf("[info]: unable to get start time!\n"); } } if (msg[i].msg_len == TEST_STOP) { double average_rx_mbytes; double average_rx_pkts; double time_diff_sec; if (clock_gettime(CLOCK_MONOTONIC, &clock_end) != 0) { printf("[info]: unable to get end time!\n"); } timespec_diff(clock_start, clock_end, &time_diff); /* * adjust for sleep(2) between sending the last data and TEST_STOP */ time_diff = time_diff - 2000000000llu; /* * convert to seconds */ time_diff_sec = (double)time_diff / 1000000000llu; average_rx_mbytes = (double)((rx_bytes / time_diff_sec) / (1024 * 1024)); average_rx_pkts = (double)(rx_pkts / time_diff_sec); if (!machine_output) { printf("[perf] execution time: %8.4f secs Average speed: %8.4f MB/sec %8.4f pckts/sec (size: %u total: %" PRIu64 ")\n", time_diff_sec, average_rx_mbytes, average_rx_pkts, current_pckt_size, rx_pkts); } else { printf("[perf],%.4f,%u,%" PRIu64 ",%.4f,%.4f\n", time_diff_sec, current_pckt_size, rx_pkts, average_rx_mbytes, average_rx_pkts); } rx_pkts = 0; rx_bytes = 0; current_pckt_size = 0; } if (msg[i].msg_len == TEST_COMPLETE) { wait_for_perf_rx = 1; } continue; } if (use_pckt_verification) { struct pckt_ver *recv_pckt = (struct pckt_ver *)msg[i].msg_hdr.msg_iov->iov_base; uint32_t chksum; if (msg[i].msg_len != recv_pckt->len) { printf("Wrong packet len received: %u expected: %u!\n", msg[i].msg_len, recv_pckt->len); exit(FAIL); } chksum = compute_chksum((const unsigned char *)msg[i].msg_hdr.msg_iov->iov_base + sizeof(struct pckt_ver), msg[i].msg_len - sizeof(struct pckt_ver)); if (recv_pckt->chksum != chksum){ printf("Wrong packet checksum received: %u expected: %u!\n", recv_pckt->chksum, chksum); exit(FAIL); } } rx_pkts++; rx_bytes = rx_bytes + msg[i].msg_len; current_pckt_size = msg[i].msg_len; } break; } } } epoll_ctl(rx_epoll, EPOLL_CTL_DEL, datafd, &ev); close(rx_epoll); return NULL; } static void setup_data_txrx_common(void) { if (!rx_thread) { if (knet_handle_enable_filter(knet_h, NULL, ping_dst_host_filter)) { printf("Unable to enable dst_host_filter: %s\n", strerror(errno)); exit(FAIL); } printf("[info]: setting up rx thread\n"); if (pthread_create(&rx_thread, 0, _rx_thread, NULL)) { printf("Unable to start rx thread\n"); exit(FAIL); } } } static void stop_rx_thread(void) { void *retval; unsigned int i; if (rx_thread) { printf("[info]: shutting down rx thread\n"); sleep(2); pthread_cancel(rx_thread); pthread_join(rx_thread, &retval); for (i = 0; i < PCKT_FRAG_MAX; i ++) { free(rx_buf[i]); } } } static void send_ping_data(void) { char buf[65535]; ssize_t len; memset(&buf, 0, sizeof(buf)); snprintf(buf, sizeof(buf), "Hello world!"); if (compresscfg) { len = sizeof(buf); } else { len = strlen(buf); } if (knet_send(knet_h, buf, len, channel) != len) { printf("[info]: Error sending hello world: %s\n", strerror(errno)); } sleep(1); } static int send_messages(struct knet_mmsghdr *msg, int msgs_to_send) { int sent_msgs, prev_sent, progress, total_sent; total_sent = 0; sent_msgs = 0; prev_sent = 0; progress = 1; retry: errno = 0; sent_msgs = _sendmmsg(datafd, 0, &msg[0], msgs_to_send, MSG_NOSIGNAL); if (sent_msgs < 0) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { usleep(KNET_THREADS_TIMER_RES / 16); goto retry; } printf("[info]: Unable to send messages: %s\n", strerror(errno)); return -1; } total_sent = total_sent + sent_msgs; if ((sent_msgs >= 0) && (sent_msgs < msgs_to_send)) { if ((sent_msgs) || (progress)) { msgs_to_send = msgs_to_send - sent_msgs; prev_sent = prev_sent + sent_msgs; if (sent_msgs) { progress = 1; } else { progress = 0; } goto retry; } if (!progress) { printf("[info]: Unable to send more messages after retry\n"); } } return total_sent; } static int setup_send_buffers_common(struct knet_mmsghdr *msg, struct iovec *iov_out, char *tx_buf[]) { unsigned int i; for (i = 0; i < PCKT_FRAG_MAX; i++) { tx_buf[i] = malloc(KNET_MAX_PACKET_SIZE); if (!tx_buf[i]) { printf("TXT: Unable to malloc!\n"); return -1; } memset(tx_buf[i], i, KNET_MAX_PACKET_SIZE); iov_out[i].iov_base = (void *)tx_buf[i]; memset(&msg[i].msg_hdr, 0, sizeof(struct msghdr)); msg[i].msg_hdr.msg_iov = &iov_out[i]; msg[i].msg_hdr.msg_iovlen = 1; } return 0; } static void send_perf_data_by_size(void) { char *tx_buf[PCKT_FRAG_MAX]; struct knet_mmsghdr msg[PCKT_FRAG_MAX]; struct iovec iov_out[PCKT_FRAG_MAX]; char ctrl_message[16]; int sent_msgs; unsigned int i; uint64_t total_pkts_to_tx; uint64_t packets_to_send; uint32_t packetsize = 64; setup_send_buffers_common(msg, iov_out, tx_buf); while (packetsize <= KNET_MAX_PACKET_SIZE) { if (force_packet_size) { packetsize = force_packet_size; } for (i = 0; i < PCKT_FRAG_MAX; i++) { iov_out[i].iov_len = packetsize; if (use_pckt_verification) { struct pckt_ver *tx_pckt = (struct pckt_ver *)&iov_out[i].iov_base; tx_pckt->len = iov_out[i].iov_len; tx_pckt->chksum = compute_chksum((const unsigned char *)iov_out[i].iov_base + sizeof(struct pckt_ver), iov_out[i].iov_len - sizeof(struct pckt_ver)); } } total_pkts_to_tx = perf_by_size_size / packetsize; printf("[info]: testing with %u packet size. total bytes to transfer: %" PRIu64 " (%" PRIu64 " packets)\n", packetsize, perf_by_size_size, total_pkts_to_tx); memset(ctrl_message, 0, sizeof(ctrl_message)); knet_send(knet_h, ctrl_message, TEST_START, channel); while (total_pkts_to_tx > 0) { if (total_pkts_to_tx >= PCKT_FRAG_MAX) { packets_to_send = PCKT_FRAG_MAX; } else { packets_to_send = total_pkts_to_tx; } sent_msgs = send_messages(&msg[0], packets_to_send); if (sent_msgs < 0) { printf("Something went wrong, aborting\n"); exit(FAIL); } total_pkts_to_tx = total_pkts_to_tx - sent_msgs; } sleep(2); knet_send(knet_h, ctrl_message, TEST_STOP, channel); if ((packetsize == KNET_MAX_PACKET_SIZE) || (force_packet_size)) { break; } /* * Use a multiplier that can always divide properly a GB * into smaller chunks without worry about boundaries */ packetsize *= 4; if (packetsize > KNET_MAX_PACKET_SIZE) { packetsize = KNET_MAX_PACKET_SIZE; } } knet_send(knet_h, ctrl_message, TEST_COMPLETE, channel); for (i = 0; i < PCKT_FRAG_MAX; i++) { free(tx_buf[i]); } } /* For sorting the node list into order */ static int node_compare(const void *aptr, const void *bptr) { uint16_t a,b; a = *(uint16_t *)aptr; b = *(uint16_t *)bptr; return a > b; } static void display_stats(int level) { struct knet_handle_stats handle_stats; struct knet_link_status link_status; struct knet_link_stats total_link_stats; knet_node_id_t host_list[KNET_MAX_HOST]; uint8_t link_list[KNET_MAX_LINK]; unsigned int i,j; size_t num_hosts, num_links; if (knet_handle_get_stats(knet_h, &handle_stats, sizeof(handle_stats)) < 0) { perror("[info]: failed to get knet handle stats"); return; } if (compresscfg || cryptocfg) { printf("\n"); printf("[stat]: handle stats\n"); printf("[stat]: ------------\n"); if (compresscfg) { printf("[stat]: tx_uncompressed_packets: %" PRIu64 "\n", handle_stats.tx_uncompressed_packets); printf("[stat]: tx_compressed_packets: %" PRIu64 "\n", handle_stats.tx_compressed_packets); printf("[stat]: tx_compressed_original_bytes: %" PRIu64 "\n", handle_stats.tx_compressed_original_bytes); printf("[stat]: tx_compressed_size_bytes: %" PRIu64 "\n", handle_stats.tx_compressed_size_bytes ); printf("[stat]: tx_compress_time_ave: %" PRIu64 "\n", handle_stats.tx_compress_time_ave); printf("[stat]: tx_compress_time_min: %" PRIu64 "\n", handle_stats.tx_compress_time_min); printf("[stat]: tx_compress_time_max: %" PRIu64 "\n", handle_stats.tx_compress_time_max); printf("[stat]: tx_failed_to_compress: %" PRIu64 "\n", handle_stats.tx_failed_to_compress); printf("[stat]: tx_unable_to_compress: %" PRIu64 "\n", handle_stats.tx_unable_to_compress); printf("[stat]: rx_compressed_packets: %" PRIu64 "\n", handle_stats.rx_compressed_packets); printf("[stat]: rx_compressed_original_bytes: %" PRIu64 "\n", handle_stats.rx_compressed_original_bytes); printf("[stat]: rx_compressed_size_bytes: %" PRIu64 "\n", handle_stats.rx_compressed_size_bytes); printf("[stat]: rx_compress_time_ave: %" PRIu64 "\n", handle_stats.rx_compress_time_ave); printf("[stat]: rx_compress_time_min: %" PRIu64 "\n", handle_stats.rx_compress_time_min); printf("[stat]: rx_compress_time_max: %" PRIu64 "\n", handle_stats.rx_compress_time_max); printf("[stat]: rx_failed_to_decompress: %" PRIu64 "\n", handle_stats.rx_failed_to_decompress); printf("\n"); } if (cryptocfg) { printf("[stat]: tx_crypt_packets: %" PRIu64 "\n", handle_stats.tx_crypt_packets); printf("[stat]: tx_crypt_byte_overhead: %" PRIu64 "\n", handle_stats.tx_crypt_byte_overhead); printf("[stat]: tx_crypt_time_ave: %" PRIu64 "\n", handle_stats.tx_crypt_time_ave); printf("[stat]: tx_crypt_time_min: %" PRIu64 "\n", handle_stats.tx_crypt_time_min); printf("[stat]: tx_crypt_time_max: %" PRIu64 "\n", handle_stats.tx_crypt_time_max); printf("[stat]: rx_crypt_packets: %" PRIu64 "\n", handle_stats.rx_crypt_packets); printf("[stat]: rx_crypt_time_ave: %" PRIu64 "\n", handle_stats.rx_crypt_time_ave); printf("[stat]: rx_crypt_time_min: %" PRIu64 "\n", handle_stats.rx_crypt_time_min); printf("[stat]: rx_crypt_time_max: %" PRIu64 "\n", handle_stats.rx_crypt_time_max); printf("\n"); } } if (level < 2) { return; } memset(&total_link_stats, 0, sizeof(struct knet_link_stats)); if (knet_host_get_host_list(knet_h, host_list, &num_hosts) < 0) { perror("[info]: cannot get host list for stats"); return; } /* Print in host ID order */ qsort(host_list, num_hosts, sizeof(uint16_t), node_compare); for (j=0; j 2) { printf("\n"); printf("[stat]: Node %d Link %d\n", host_list[j], link_list[i]); printf("[stat]: tx_data_packets: %" PRIu64 "\n", link_status.stats.tx_data_packets); printf("[stat]: rx_data_packets: %" PRIu64 "\n", link_status.stats.rx_data_packets); printf("[stat]: tx_data_bytes: %" PRIu64 "\n", link_status.stats.tx_data_bytes); printf("[stat]: rx_data_bytes: %" PRIu64 "\n", link_status.stats.rx_data_bytes); printf("[stat]: rx_ping_packets: %" PRIu64 "\n", link_status.stats.rx_ping_packets); printf("[stat]: tx_ping_packets: %" PRIu64 "\n", link_status.stats.tx_ping_packets); printf("[stat]: rx_ping_bytes: %" PRIu64 "\n", link_status.stats.rx_ping_bytes); printf("[stat]: tx_ping_bytes: %" PRIu64 "\n", link_status.stats.tx_ping_bytes); printf("[stat]: rx_pong_packets: %" PRIu64 "\n", link_status.stats.rx_pong_packets); printf("[stat]: tx_pong_packets: %" PRIu64 "\n", link_status.stats.tx_pong_packets); printf("[stat]: rx_pong_bytes: %" PRIu64 "\n", link_status.stats.rx_pong_bytes); printf("[stat]: tx_pong_bytes: %" PRIu64 "\n", link_status.stats.tx_pong_bytes); printf("[stat]: rx_pmtu_packets: %" PRIu64 "\n", link_status.stats.rx_pmtu_packets); printf("[stat]: tx_pmtu_packets: %" PRIu64 "\n", link_status.stats.tx_pmtu_packets); printf("[stat]: rx_pmtu_bytes: %" PRIu64 "\n", link_status.stats.rx_pmtu_bytes); printf("[stat]: tx_pmtu_bytes: %" PRIu64 "\n", link_status.stats.tx_pmtu_bytes); printf("[stat]: tx_total_packets: %" PRIu64 "\n", link_status.stats.tx_total_packets); printf("[stat]: rx_total_packets: %" PRIu64 "\n", link_status.stats.rx_total_packets); printf("[stat]: tx_total_bytes: %" PRIu64 "\n", link_status.stats.tx_total_bytes); printf("[stat]: rx_total_bytes: %" PRIu64 "\n", link_status.stats.rx_total_bytes); printf("[stat]: tx_total_errors: %" PRIu64 "\n", link_status.stats.tx_total_errors); printf("[stat]: tx_total_retries: %" PRIu64 "\n", link_status.stats.tx_total_retries); printf("[stat]: tx_pmtu_errors: %" PRIu32 "\n", link_status.stats.tx_pmtu_errors); printf("[stat]: tx_pmtu_retries: %" PRIu32 "\n", link_status.stats.tx_pmtu_retries); printf("[stat]: tx_ping_errors: %" PRIu32 "\n", link_status.stats.tx_ping_errors); printf("[stat]: tx_ping_retries: %" PRIu32 "\n", link_status.stats.tx_ping_retries); printf("[stat]: tx_pong_errors: %" PRIu32 "\n", link_status.stats.tx_pong_errors); printf("[stat]: tx_pong_retries: %" PRIu32 "\n", link_status.stats.tx_pong_retries); printf("[stat]: tx_data_errors: %" PRIu32 "\n", link_status.stats.tx_data_errors); printf("[stat]: tx_data_retries: %" PRIu32 "\n", link_status.stats.tx_data_retries); printf("[stat]: latency_min: %" PRIu32 "\n", link_status.stats.latency_min); printf("[stat]: latency_max: %" PRIu32 "\n", link_status.stats.latency_max); printf("[stat]: latency_ave: %" PRIu32 "\n", link_status.stats.latency_ave); printf("[stat]: latency_samples: %" PRIu32 "\n", link_status.stats.latency_samples); printf("[stat]: down_count: %" PRIu32 "\n", link_status.stats.down_count); printf("[stat]: up_count: %" PRIu32 "\n", link_status.stats.up_count); } } } printf("\n"); printf("[stat]: Total link stats\n"); printf("[stat]: ----------------\n"); printf("[stat]: tx_data_packets: %" PRIu64 "\n", total_link_stats.tx_data_packets); printf("[stat]: rx_data_packets: %" PRIu64 "\n", total_link_stats.rx_data_packets); printf("[stat]: tx_data_bytes: %" PRIu64 "\n", total_link_stats.tx_data_bytes); printf("[stat]: rx_data_bytes: %" PRIu64 "\n", total_link_stats.rx_data_bytes); printf("[stat]: rx_ping_packets: %" PRIu64 "\n", total_link_stats.rx_ping_packets); printf("[stat]: tx_ping_packets: %" PRIu64 "\n", total_link_stats.tx_ping_packets); printf("[stat]: rx_ping_bytes: %" PRIu64 "\n", total_link_stats.rx_ping_bytes); printf("[stat]: tx_ping_bytes: %" PRIu64 "\n", total_link_stats.tx_ping_bytes); printf("[stat]: rx_pong_packets: %" PRIu64 "\n", total_link_stats.rx_pong_packets); printf("[stat]: tx_pong_packets: %" PRIu64 "\n", total_link_stats.tx_pong_packets); printf("[stat]: rx_pong_bytes: %" PRIu64 "\n", total_link_stats.rx_pong_bytes); printf("[stat]: tx_pong_bytes: %" PRIu64 "\n", total_link_stats.tx_pong_bytes); printf("[stat]: rx_pmtu_packets: %" PRIu64 "\n", total_link_stats.rx_pmtu_packets); printf("[stat]: tx_pmtu_packets: %" PRIu64 "\n", total_link_stats.tx_pmtu_packets); printf("[stat]: rx_pmtu_bytes: %" PRIu64 "\n", total_link_stats.rx_pmtu_bytes); printf("[stat]: tx_pmtu_bytes: %" PRIu64 "\n", total_link_stats.tx_pmtu_bytes); printf("[stat]: tx_total_packets: %" PRIu64 "\n", total_link_stats.tx_total_packets); printf("[stat]: rx_total_packets: %" PRIu64 "\n", total_link_stats.rx_total_packets); printf("[stat]: tx_total_bytes: %" PRIu64 "\n", total_link_stats.tx_total_bytes); printf("[stat]: rx_total_bytes: %" PRIu64 "\n", total_link_stats.rx_total_bytes); printf("[stat]: tx_total_errors: %" PRIu64 "\n", total_link_stats.tx_total_errors); printf("[stat]: tx_total_retries: %" PRIu64 "\n", total_link_stats.tx_total_retries); printf("[stat]: tx_pmtu_errors: %" PRIu32 "\n", total_link_stats.tx_pmtu_errors); printf("[stat]: tx_pmtu_retries: %" PRIu32 "\n", total_link_stats.tx_pmtu_retries); printf("[stat]: tx_ping_errors: %" PRIu32 "\n", total_link_stats.tx_ping_errors); printf("[stat]: tx_ping_retries: %" PRIu32 "\n", total_link_stats.tx_ping_retries); printf("[stat]: tx_pong_errors: %" PRIu32 "\n", total_link_stats.tx_pong_errors); printf("[stat]: tx_pong_retries: %" PRIu32 "\n", total_link_stats.tx_pong_retries); printf("[stat]: tx_data_errors: %" PRIu32 "\n", total_link_stats.tx_data_errors); printf("[stat]: tx_data_retries: %" PRIu32 "\n", total_link_stats.tx_data_retries); printf("[stat]: down_count: %" PRIu32 "\n", total_link_stats.down_count); printf("[stat]: up_count: %" PRIu32 "\n", total_link_stats.up_count); } static void send_perf_data_by_time(void) { char *tx_buf[PCKT_FRAG_MAX]; struct knet_mmsghdr msg[PCKT_FRAG_MAX]; struct iovec iov_out[PCKT_FRAG_MAX]; char ctrl_message[16]; int sent_msgs; unsigned int i; uint32_t packetsize = 64; struct timespec clock_start, clock_end; unsigned long long time_diff = 0; setup_send_buffers_common(msg, iov_out, tx_buf); memset(&clock_start, 0, sizeof(clock_start)); memset(&clock_end, 0, sizeof(clock_start)); while (packetsize <= KNET_MAX_PACKET_SIZE) { if (force_packet_size) { packetsize = force_packet_size; } for (i = 0; i < PCKT_FRAG_MAX; i++) { iov_out[i].iov_len = packetsize; if (use_pckt_verification) { struct pckt_ver *tx_pckt = (struct pckt_ver *)iov_out[i].iov_base; tx_pckt->len = iov_out[i].iov_len; tx_pckt->chksum = compute_chksum((const unsigned char *)iov_out[i].iov_base + sizeof(struct pckt_ver), iov_out[i].iov_len - sizeof(struct pckt_ver)); } } printf("[info]: testing with %u bytes packet size for %" PRIu64 " seconds.\n", packetsize, perf_by_time_secs); memset(ctrl_message, 0, sizeof(ctrl_message)); knet_send(knet_h, ctrl_message, TEST_START, channel); if (clock_gettime(CLOCK_MONOTONIC, &clock_start) != 0) { printf("[info]: unable to get start time!\n"); } time_diff = 0; while (time_diff < (perf_by_time_secs * 1000000000llu)) { sent_msgs = send_messages(&msg[0], PCKT_FRAG_MAX); if (sent_msgs < 0) { printf("Something went wrong, aborting\n"); exit(FAIL); } if (clock_gettime(CLOCK_MONOTONIC, &clock_end) != 0) { printf("[info]: unable to get end time!\n"); } timespec_diff(clock_start, clock_end, &time_diff); } sleep(2); knet_send(knet_h, ctrl_message, TEST_STOP, channel); if ((packetsize == KNET_MAX_PACKET_SIZE) || (force_packet_size)) { break; } /* * Use a multiplier that can always divide properly a GB * into smaller chunks without worry about boundaries */ packetsize *= 4; if (packetsize > KNET_MAX_PACKET_SIZE) { packetsize = KNET_MAX_PACKET_SIZE; } } knet_send(knet_h, ctrl_message, TEST_COMPLETE, channel); for (i = 0; i < PCKT_FRAG_MAX; i++) { free(tx_buf[i]); } } static void cleanup_all(void) { knet_handle_t knet_h_tmp[2]; if (pthread_mutex_lock(&shutdown_mutex)) { return; } if (bench_shutdown_in_progress) { pthread_mutex_unlock(&shutdown_mutex); return; } bench_shutdown_in_progress = 1; pthread_mutex_unlock(&shutdown_mutex); if (rx_thread) { stop_rx_thread(); } knet_h_tmp[1] = knet_h; knet_handle_stop_everything(knet_h_tmp, 1); } static void sigint_handler(int signum) { printf("[info]: cleaning up... got signal: %d\n", signum); cleanup_all(); exit(PASS); } int main(int argc, char *argv[]) { if (signal(SIGINT, sigint_handler) == SIG_ERR) { printf("Unable to configure SIGINT handler\n"); exit(FAIL); } setup_knet(argc, argv); setup_data_txrx_common(); sleep(5); restart: switch(test_type) { default: case TEST_PING: /* basic ping, no data */ sleep(5); break; case TEST_PING_AND_DATA: send_ping_data(); break; case TEST_PERF_BY_SIZE: if (senderid == thisnodeid) { send_perf_data_by_size(); } else { printf("[info]: waiting for perf rx thread to finish\n"); while(!wait_for_perf_rx) { sleep(1); } } break; case TEST_PERF_BY_TIME: if (senderid == thisnodeid) { send_perf_data_by_time(); } else { printf("[info]: waiting for perf rx thread to finish\n"); while(!wait_for_perf_rx) { sleep(1); } } break; } if (continous) { goto restart; } if (show_stats) { display_stats(show_stats); } cleanup_all(); return PASS; } diff --git a/libknet/threads_common.c b/libknet/threads_common.c index 6a31c812..0312b0a5 100644 --- a/libknet/threads_common.c +++ b/libknet/threads_common.c @@ -1,383 +1,379 @@ /* * Copyright (C) 2016-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under LGPL-2.0+ */ #include "config.h" #include #include #include #include #include "internals.h" #include "logging.h" #include "threads_common.h" int shutdown_in_progress(knet_handle_t knet_h) { int savederrno = 0; int ret; savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_COMMON, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } ret = knet_h->fini_in_progress; pthread_rwlock_unlock(&knet_h->global_rwlock); return ret; } static int _pmtud_reschedule(knet_handle_t knet_h) { if (knet_h->pmtud_running) { knet_h->pmtud_abort = 1; if (knet_h->pmtud_waiting) { pthread_cond_signal(&knet_h->pmtud_cond); } } return 0; } static int pmtud_reschedule(knet_handle_t knet_h) { int res; if (pthread_mutex_lock(&knet_h->pmtud_mutex) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock"); return -1; } res = _pmtud_reschedule(knet_h); pthread_mutex_unlock(&knet_h->pmtud_mutex); return res; } int get_global_wrlock(knet_handle_t knet_h) { if (pmtud_reschedule(knet_h) < 0) { log_info(knet_h, KNET_SUB_PMTUD, "Unable to notify PMTUd to reschedule. Expect delays in executing API calls"); } return pthread_rwlock_wrlock(&knet_h->global_rwlock); } static struct pretty_names thread_names[KNET_THREAD_MAX] = { { "TX", KNET_THREAD_TX }, { "RX", KNET_THREAD_RX }, { "HB", KNET_THREAD_HB }, { "PMTUD", KNET_THREAD_PMTUD }, -#ifdef HAVE_NETINET_SCTP_H - { "SCTP_LISTEN", KNET_THREAD_SCTP_LISTEN }, - { "SCTP_CONN", KNET_THREAD_SCTP_CONN }, -#endif { "DST_LINK", KNET_THREAD_DST_LINK } }; static struct pretty_names thread_status[] = { { "unregistered", KNET_THREAD_UNREGISTERED }, { "registered", KNET_THREAD_REGISTERED }, { "started", KNET_THREAD_STARTED }, { "stopped", KNET_THREAD_STOPPED } }; static const char *get_thread_status_name(uint8_t status) { unsigned int i; for (i = 0; i < KNET_THREAD_STATUS_MAX; i++) { if (thread_status[i].val == status) { return thread_status[i].name; } } return "unknown"; } static const char *get_thread_name(uint8_t thread_id) { unsigned int i; for (i = 0; i < KNET_THREAD_MAX; i++) { if (thread_names[i].val == thread_id) { return thread_names[i].name; } } return "unknown"; } int get_thread_flush_queue(knet_handle_t knet_h, uint8_t thread_id) { uint8_t flush; if (pthread_mutex_lock(&knet_h->threads_status_mutex) != 0) { log_debug(knet_h, KNET_SUB_HANDLE, "Unable to get mutex lock"); return -1; } flush = knet_h->threads_flush_queue[thread_id]; pthread_mutex_unlock(&knet_h->threads_status_mutex); return flush; } int set_thread_flush_queue(knet_handle_t knet_h, uint8_t thread_id, uint8_t status) { if (pthread_mutex_lock(&knet_h->threads_status_mutex) != 0) { log_debug(knet_h, KNET_SUB_HANDLE, "Unable to get mutex lock"); return -1; } knet_h->threads_flush_queue[thread_id] = status; log_debug(knet_h, KNET_SUB_HANDLE, "Updated flush queue request for thread %s to %u", get_thread_name(thread_id), status); pthread_mutex_unlock(&knet_h->threads_status_mutex); return 0; } int wait_all_threads_flush_queue(knet_handle_t knet_h) { uint8_t i = 0, found = 0; while (!found) { usleep(knet_h->threads_timer_res); if (pthread_mutex_lock(&knet_h->threads_status_mutex) != 0) { continue; } found = 1; for (i = 0; i < KNET_THREAD_MAX; i++) { if (knet_h->threads_flush_queue[i] == KNET_THREAD_QUEUE_FLUSHED) { continue; } log_debug(knet_h, KNET_SUB_HANDLE, "Checking thread: %s queue: %u", get_thread_name(i), knet_h->threads_flush_queue[i]); if (knet_h->threads_flush_queue[i] != KNET_THREAD_QUEUE_FLUSHED) { found = 0; } } pthread_mutex_unlock(&knet_h->threads_status_mutex); } return 0; } int set_thread_status(knet_handle_t knet_h, uint8_t thread_id, uint8_t status) { if (pthread_mutex_lock(&knet_h->threads_status_mutex) != 0) { log_debug(knet_h, KNET_SUB_HANDLE, "Unable to get mutex lock"); return -1; } knet_h->threads_status[thread_id] = status; log_debug(knet_h, KNET_SUB_HANDLE, "Updated status for thread %s to %s", get_thread_name(thread_id), get_thread_status_name(status)); pthread_mutex_unlock(&knet_h->threads_status_mutex); return 0; } int wait_all_threads_status(knet_handle_t knet_h, uint8_t status) { uint8_t i = 0, found = 0; while (!found) { usleep(knet_h->threads_timer_res); if (pthread_mutex_lock(&knet_h->threads_status_mutex) != 0) { continue; } found = 1; for (i = 0; i < KNET_THREAD_MAX; i++) { if (knet_h->threads_status[i] == KNET_THREAD_UNREGISTERED) { continue; } log_debug(knet_h, KNET_SUB_HANDLE, "Checking thread: %s status: %s req: %s", get_thread_name(i), get_thread_status_name(knet_h->threads_status[i]), get_thread_status_name(status)); if (knet_h->threads_status[i] != status) { found = 0; } } pthread_mutex_unlock(&knet_h->threads_status_mutex); } return 0; } void force_pmtud_run(knet_handle_t knet_h, uint8_t subsystem, uint8_t reset_mtu, uint8_t force_restart) { if (reset_mtu) { log_debug(knet_h, subsystem, "PMTUd has been reset to default"); knet_h->data_mtu = calc_min_mtu(knet_h); if (knet_h->pmtud_notify_fn) { knet_h->pmtud_notify_fn(knet_h->pmtud_notify_fn_private_data, knet_h->data_mtu); } } /* * we can only try to take a lock here. This part of the code * can be invoked by any thread, including PMTUd that is already * holding a lock at that stage. * If PMTUd is holding the lock, most likely it is already running * and we don't need to notify it back. */ if (!pthread_mutex_trylock(&knet_h->pmtud_mutex)) { if (!knet_h->pmtud_running) { if (!knet_h->pmtud_forcerun) { log_debug(knet_h, subsystem, "Notifying PMTUd to rerun"); knet_h->pmtud_forcerun = 1; } } else { if (force_restart) { if (_pmtud_reschedule(knet_h) < 0) { log_info(knet_h, KNET_SUB_PMTUD, "Unable to notify PMTUd to reschedule. A joining node may struggle to connect properly"); } } } pthread_mutex_unlock(&knet_h->pmtud_mutex); } } int knet_handle_set_threads_timer_res(knet_handle_t knet_h, useconds_t timeres) { int savederrno = 0, err = 0; struct knet_host *host; int link_idx; int found = 0; if (!_is_valid_handle(knet_h)) { return -1; } /* * most threads use timeres / 1000 as timeout on epoll. * anything below 1000 would generate a result of 0, making * the threads spin at 100% cpu */ if ((timeres > 0) && (timeres < 1000)) { errno = EINVAL; return -1; } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (timeres) { if (timeres > knet_h->threads_timer_res) { for (host = knet_h->host_head; host != NULL; host = host->next) { for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) { if (!host->link[link_idx].configured) { continue; } if (host->link[link_idx].ping_interval < timeres) { log_warn(knet_h, KNET_SUB_HANDLE, "Requested new threads timer resolution %u is higher than detected link ping interval on host: %u link: %u interval: %llu (ns).", timeres, host->host_id, link_idx, host->link[link_idx].ping_interval); } if (host->link[link_idx].pong_timeout < timeres) { log_err(knet_h, KNET_SUB_HANDLE, "Requested new threads timer resolution %u is higher than detected link pong time on host: %u link: %u timeout: %llu (ns) and will cause network instability", timeres, host->host_id, link_idx, host->link[link_idx].pong_timeout); found = 1; } } } if (found) { err = -1; savederrno = EINVAL; goto exit_unlock; } } knet_h->threads_timer_res = timeres; log_debug(knet_h, KNET_SUB_HANDLE, "Setting new threads timer resolution to %u usecs", knet_h->threads_timer_res); } else { knet_h->threads_timer_res = KNET_THREADS_TIMER_RES; log_debug(knet_h, KNET_SUB_HANDLE, "Setting new threads timer resolution to default %u usecs", knet_h->threads_timer_res); } exit_unlock: pthread_rwlock_unlock(&knet_h->global_rwlock); errno = err ? savederrno : 0; return err; } int knet_handle_get_threads_timer_res(knet_handle_t knet_h, useconds_t *timeres) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (!timeres) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } *timeres = knet_h->threads_timer_res; pthread_rwlock_unlock(&knet_h->global_rwlock); return 0; } uint32_t compute_chksum(const unsigned char *data, uint32_t data_len) { uLong crc; crc = crc32(0, NULL, 0); crc = crc32(crc, (Bytef*)data, data_len); return crc; } uint32_t compute_chksumv(const struct iovec *iov_in, int iovcnt_in) { uLong crc; int i; crc = crc32(0, NULL, 0); for (i = 0; i < iovcnt_in; i++) { crc = crc32(crc, (Bytef*)iov_in[i].iov_base, iov_in[i].iov_len); } return crc; } diff --git a/libknet/threads_common.h b/libknet/threads_common.h index f2bd55be..8d7faf0c 100644 --- a/libknet/threads_common.h +++ b/libknet/threads_common.h @@ -1,56 +1,52 @@ /* * Copyright (C) 2012-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under LGPL-2.0+ */ #ifndef __KNET_THREADS_COMMON_H__ #define __KNET_THREADS_COMMON_H__ #include "internals.h" #define KNET_THREAD_UNREGISTERED 0 /* thread does not exist */ #define KNET_THREAD_REGISTERED 1 /* thread has been registered before pthread_create invocation. make sure threads are registered before calling wait_all_thread_status */ #define KNET_THREAD_STARTED 2 /* thread has reported to be running */ #define KNET_THREAD_STOPPED 3 /* thread has returned */ #define KNET_THREAD_STATUS_MAX KNET_THREAD_STOPPED + 1 #define KNET_THREAD_TX 0 #define KNET_THREAD_RX 1 #define KNET_THREAD_HB 2 #define KNET_THREAD_PMTUD 3 #define KNET_THREAD_DST_LINK 4 -#ifdef HAVE_NETINET_SCTP_H -#define KNET_THREAD_SCTP_LISTEN 5 -#define KNET_THREAD_SCTP_CONN 6 -#endif #define KNET_THREAD_MAX 32 #define KNET_THREAD_QUEUE_FLUSHED 0 #define KNET_THREAD_QUEUE_FLUSH 1 #define timespec_diff(start, end, diff) \ do { \ if (end.tv_sec > start.tv_sec) \ *(diff) = ((end.tv_sec - start.tv_sec) * 1000000000llu) \ + end.tv_nsec - start.tv_nsec; \ else \ *(diff) = end.tv_nsec - start.tv_nsec; \ } while (0); int shutdown_in_progress(knet_handle_t knet_h); int get_global_wrlock(knet_handle_t knet_h); int get_thread_flush_queue(knet_handle_t knet_h, uint8_t thread_id); int set_thread_flush_queue(knet_handle_t knet_h, uint8_t thread_id, uint8_t status); int wait_all_threads_flush_queue(knet_handle_t knet_h); int set_thread_status(knet_handle_t knet_h, uint8_t thread_id, uint8_t status); int wait_all_threads_status(knet_handle_t knet_h, uint8_t status); void force_pmtud_run(knet_handle_t knet_h, uint8_t subsystem, uint8_t reset_mtu, uint8_t force_restart); uint32_t compute_chksum(const unsigned char *data, uint32_t data_len); uint32_t compute_chksumv(const struct iovec *iov_in, int iovcnt_in); #endif diff --git a/libknet/threads_pmtud.c b/libknet/threads_pmtud.c index 116d66f6..f32facb4 100644 --- a/libknet/threads_pmtud.c +++ b/libknet/threads_pmtud.c @@ -1,938 +1,938 @@ /* * Copyright (C) 2015-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under LGPL-2.0+ */ #include "config.h" #include #include #include #include #include "crypto.h" #include "links.h" #include "host.h" #include "logging.h" #include "transports.h" #include "threads_common.h" #include "threads_pmtud.h" #include "onwire_v1.h" static int _calculate_manual_mtu(knet_handle_t knet_h, struct knet_link *dst_link) { size_t ipproto_overhead_len; /* onwire packet overhead (protocol based) */ switch (dst_link->dst_addr.ss_family) { case AF_INET6: ipproto_overhead_len = KNET_PMTUD_OVERHEAD_V6 + dst_link->proto_overhead; break; case AF_INET: ipproto_overhead_len = KNET_PMTUD_OVERHEAD_V4 + dst_link->proto_overhead; break; default: log_debug(knet_h, KNET_SUB_PMTUD, "unknown protocol"); return 0; break; } dst_link->status.mtu = calc_max_data_outlen(knet_h, knet_h->manual_mtu - ipproto_overhead_len); return 1; } static int _handle_check_link_pmtud(knet_handle_t knet_h, struct knet_host *dst_host, struct knet_link *dst_link) { int err, ret, savederrno, mutex_retry_limit, failsafe, use_kernel_mtu, warn_once; uint32_t kernel_mtu; /* record kernel_mtu from EMSGSIZE */ size_t onwire_len; /* current packet onwire size */ size_t ipproto_overhead_len; /* onwire packet overhead (protocol based) */ size_t max_mtu_len; /* max mtu for protocol */ size_t data_len; /* how much data we can send in the packet * generally would be onwire_len - ipproto_overhead_len * needs to be adjusted for crypto */ size_t app_mtu_len; /* real data that we can send onwire */ ssize_t len; /* len of what we were able to sendto onwire */ uint8_t onwire_ver; struct timespec ts, pmtud_crypto_start_ts, pmtud_crypto_stop_ts; unsigned long long pong_timeout_adj_tmp, timediff; int pmtud_crypto_reduce = 1; unsigned char *outbuf = (unsigned char *)knet_h->pmtudbuf; warn_once = 0; mutex_retry_limit = 0; failsafe = 0; switch (dst_link->dst_addr.ss_family) { case AF_INET6: max_mtu_len = KNET_PMTUD_SIZE_V6; ipproto_overhead_len = KNET_PMTUD_OVERHEAD_V6 + dst_link->proto_overhead; break; case AF_INET: max_mtu_len = KNET_PMTUD_SIZE_V4; ipproto_overhead_len = KNET_PMTUD_OVERHEAD_V4 + dst_link->proto_overhead; break; default: log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD aborted, unknown protocol"); return -1; break; } dst_link->last_bad_mtu = 0; dst_link->last_good_mtu = dst_link->last_ping_size + ipproto_overhead_len; /* * discovery starts from the top because kernel will * refuse to send packets > current iface mtu. * this saves us some time and network bw. */ onwire_len = max_mtu_len; /* * cache onwire version for this link / run */ if (pthread_mutex_lock(&knet_h->onwire_mutex)) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get onwire mutex lock"); return -1; } onwire_ver = knet_h->onwire_ver; pthread_mutex_unlock(&knet_h->onwire_mutex); restart: /* * prevent a race when interface mtu is changed _exactly_ during * the discovery process and it's complex to detect. Easier * to wait the next loop. * 30 is not an arbitrary value. To bisect from 576 to 128000 doesn't * take more than 18/19 steps. */ if (failsafe == 30) { log_err(knet_h, KNET_SUB_PMTUD, "Aborting PMTUD process: Too many attempts. MTU might have changed during discovery."); return -1; } else { failsafe++; } /* * common to all packets */ /* * calculate the application MTU based on current onwire_len minus ipproto_overhead_len */ app_mtu_len = calc_max_data_outlen(knet_h, onwire_len - ipproto_overhead_len); /* * recalculate onwire len back that might be different based * on data padding from crypto layer. */ onwire_len = calc_data_outlen(knet_h, app_mtu_len + KNET_HEADER_ALL_SIZE) + ipproto_overhead_len; /* * calculate the size of what we need to send to sendto(2). * see also onwire.c for packet format explanation. */ data_len = app_mtu_len + knet_h->sec_hash_size + knet_h->sec_salt_size + KNET_HEADER_ALL_SIZE; if (knet_h->onwire_ver_remap) { prep_pmtud_v1(knet_h, dst_link, onwire_ver, onwire_len, app_mtu_len + KNET_HEADER_ALL_SIZE); } else { switch (onwire_ver) { case 1: prep_pmtud_v1(knet_h, dst_link, onwire_ver, onwire_len, app_mtu_len + KNET_HEADER_ALL_SIZE); break; default: log_warn(knet_h, KNET_SUB_PMTUD, "preparing PMTUD onwire version %u not supported", onwire_ver); return -1; break; } } if (knet_h->crypto_in_use_config) { if (data_len < (knet_h->sec_hash_size + knet_h->sec_salt_size) + 1) { log_debug(knet_h, KNET_SUB_PMTUD, "Aborting PMTUD process: link mtu smaller than crypto header detected (link might have been disconnected)"); return -1; } if (crypto_encrypt_and_sign(knet_h, (const unsigned char *)knet_h->pmtudbuf, data_len - (knet_h->sec_hash_size + knet_h->sec_salt_size), knet_h->pmtudbuf_crypt, (ssize_t *)&data_len) < 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to crypto pmtud packet"); return -1; } outbuf = knet_h->pmtudbuf_crypt; if (pthread_mutex_lock(&knet_h->handle_stats_mutex) < 0) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock"); return -1; } knet_h->stats_extra.tx_crypt_pmtu_packets++; pthread_mutex_unlock(&knet_h->handle_stats_mutex); } /* link has gone down, aborting pmtud */ if (dst_link->status.connected != 1) { log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD detected host (%u) link (%u) has been disconnected", dst_host->host_id, dst_link->link_id); return -1; } if (dst_link->transport_connected != 1) { log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD detected host (%u) link (%u) has been disconnected", dst_host->host_id, dst_link->link_id); return -1; } if (pthread_mutex_lock(&knet_h->pmtud_mutex) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock"); return -1; } if (knet_h->pmtud_abort) { pthread_mutex_unlock(&knet_h->pmtud_mutex); errno = EDEADLK; return -1; } savederrno = pthread_mutex_lock(&knet_h->tx_mutex); if (savederrno) { pthread_mutex_unlock(&knet_h->pmtud_mutex); log_err(knet_h, KNET_SUB_PMTUD, "Unable to get TX mutex lock: %s", strerror(savederrno)); return -1; } savederrno = pthread_mutex_lock(&dst_link->link_stats_mutex); if (savederrno) { pthread_mutex_unlock(&knet_h->pmtud_mutex); pthread_mutex_unlock(&knet_h->tx_mutex); log_err(knet_h, KNET_SUB_PMTUD, "Unable to get stats mutex lock for host %u link %u: %s", dst_host->host_id, dst_link->link_id, strerror(savederrno)); return -1; } retry: if (transport_get_connection_oriented(knet_h, dst_link->transport) == TRANSPORT_PROTO_NOT_CONNECTION_ORIENTED) { len = sendto(dst_link->outsock, outbuf, data_len, MSG_DONTWAIT | MSG_NOSIGNAL, (struct sockaddr *) &dst_link->dst_addr, knet_h->knet_transport_fd_tracker[dst_link->outsock].sockaddr_len); } else { len = sendto(dst_link->outsock, outbuf, data_len, MSG_DONTWAIT | MSG_NOSIGNAL, NULL, 0); } savederrno = errno; /* * we cannot hold a lock on kmtu_mutex between resetting * knet_h->kernel_mtu here and below where it's used. * use_kernel_mtu tells us if the knet_h->kernel_mtu was * set to 0 and we can trust its value later. */ use_kernel_mtu = 0; if (pthread_mutex_lock(&knet_h->kmtu_mutex) == 0) { use_kernel_mtu = 1; knet_h->kernel_mtu = 0; pthread_mutex_unlock(&knet_h->kmtu_mutex); } kernel_mtu = 0; err = transport_tx_sock_error(knet_h, dst_link->transport, dst_link->outsock, KNET_SUB_PMTUD, len, savederrno); switch(err) { case KNET_TRANSPORT_SOCK_ERROR_INTERNAL: log_debug(knet_h, KNET_SUB_PMTUD, "Unable to send pmtu packet (sendto): %d %s", savederrno, strerror(savederrno)); pthread_mutex_unlock(&knet_h->tx_mutex); pthread_mutex_unlock(&knet_h->pmtud_mutex); dst_link->status.stats.tx_pmtu_errors++; pthread_mutex_unlock(&dst_link->link_stats_mutex); return -1; break; case KNET_TRANSPORT_SOCK_ERROR_IGNORE: break; case KNET_TRANSPORT_SOCK_ERROR_RETRY: dst_link->status.stats.tx_pmtu_retries++; goto retry; break; } pthread_mutex_unlock(&knet_h->tx_mutex); if (len != (ssize_t )data_len) { pthread_mutex_unlock(&dst_link->link_stats_mutex); if (savederrno == EMSGSIZE || savederrno == EPERM) { /* * we cannot hold a lock on kmtu_mutex between resetting * knet_h->kernel_mtu and here. * use_kernel_mtu tells us if the knet_h->kernel_mtu was * set to 0 previously and we can trust its value now. */ if (use_kernel_mtu) { use_kernel_mtu = 0; if (pthread_mutex_lock(&knet_h->kmtu_mutex) == 0) { kernel_mtu = knet_h->kernel_mtu; pthread_mutex_unlock(&knet_h->kmtu_mutex); } } if (kernel_mtu > 0) { dst_link->last_bad_mtu = kernel_mtu + 1; } else { dst_link->last_bad_mtu = onwire_len; } } else { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to send pmtu packet len: %zu err: %s", onwire_len, strerror(savederrno)); } } else { dst_link->last_sent_mtu = onwire_len; dst_link->last_recv_mtu = 0; dst_link->status.stats.tx_pmtu_packets++; dst_link->status.stats.tx_pmtu_bytes += data_len; pthread_mutex_unlock(&dst_link->link_stats_mutex); if (clock_gettime(CLOCK_REALTIME, &ts) < 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get current time: %s", strerror(errno)); pthread_mutex_unlock(&knet_h->pmtud_mutex); return -1; } /* * non fatal, we can wait the next round to reduce the * multiplier */ if (clock_gettime(CLOCK_MONOTONIC, &pmtud_crypto_start_ts) < 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get current time: %s", strerror(errno)); pmtud_crypto_reduce = 0; } /* * set PMTUd reply timeout to match pong_timeout on a given link * * math: internally pong_timeout is expressed in microseconds, while * the public API exports milliseconds. So careful with the 0's here. * the loop is necessary because we are grabbing the current time just above * and add values to it that could overflow into seconds. */ if (pthread_mutex_lock(&knet_h->backoff_mutex)) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get backoff_mutex"); pthread_mutex_unlock(&knet_h->pmtud_mutex); return -1; } if (knet_h->crypto_in_use_config) { /* * crypto, under pressure, is a royal PITA */ pong_timeout_adj_tmp = dst_link->pong_timeout_adj * dst_link->pmtud_crypto_timeout_multiplier; } else { pong_timeout_adj_tmp = dst_link->pong_timeout_adj; } ts.tv_sec += pong_timeout_adj_tmp / 1000000; ts.tv_nsec += (((pong_timeout_adj_tmp) % 1000000) * 1000); while (ts.tv_nsec > 1000000000) { ts.tv_sec += 1; ts.tv_nsec -= 1000000000; } pthread_mutex_unlock(&knet_h->backoff_mutex); knet_h->pmtud_waiting = 1; ret = pthread_cond_timedwait(&knet_h->pmtud_cond, &knet_h->pmtud_mutex, &ts); knet_h->pmtud_waiting = 0; if (knet_h->pmtud_abort) { pthread_mutex_unlock(&knet_h->pmtud_mutex); errno = EDEADLK; return -1; } /* * we cannot use shutdown_in_progress in here because * we already hold the read lock */ if (knet_h->fini_in_progress) { pthread_mutex_unlock(&knet_h->pmtud_mutex); log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD aborted. shutdown in progress"); return -1; } if (ret) { if (ret == ETIMEDOUT) { if ((knet_h->crypto_in_use_config) && (dst_link->pmtud_crypto_timeout_multiplier < KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MAX)) { dst_link->pmtud_crypto_timeout_multiplier = dst_link->pmtud_crypto_timeout_multiplier * 2; pmtud_crypto_reduce = 0; log_debug(knet_h, KNET_SUB_PMTUD, "Increasing PMTUd response timeout multiplier to (%u) for host %u link: %u", dst_link->pmtud_crypto_timeout_multiplier, dst_host->host_id, dst_link->link_id); pthread_mutex_unlock(&knet_h->pmtud_mutex); goto restart; } if (!warn_once) { log_warn(knet_h, KNET_SUB_PMTUD, "possible MTU misconfiguration detected. " "kernel is reporting MTU: %u bytes for " "host %u link %u but the other node is " "not acknowledging packets of this size. ", dst_link->last_sent_mtu, dst_host->host_id, dst_link->link_id); log_warn(knet_h, KNET_SUB_PMTUD, "This can be caused by this node interface MTU " "too big or a network device that does not " "support or has been misconfigured to manage MTU " "of this size, or packet loss. knet will continue " "to run but performances might be affected."); warn_once = 1; } } else { pthread_mutex_unlock(&knet_h->pmtud_mutex); if (mutex_retry_limit == 3) { log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD aborted, unable to get mutex lock"); return -1; } mutex_retry_limit++; goto restart; } } if ((knet_h->crypto_in_use_config) && (pmtud_crypto_reduce == 1) && (dst_link->pmtud_crypto_timeout_multiplier > KNET_LINK_PMTUD_CRYPTO_TIMEOUT_MULTIPLIER_MIN)) { if (!clock_gettime(CLOCK_MONOTONIC, &pmtud_crypto_stop_ts)) { timespec_diff(pmtud_crypto_start_ts, pmtud_crypto_stop_ts, &timediff); if (((pong_timeout_adj_tmp * 1000) / 2) > timediff) { dst_link->pmtud_crypto_timeout_multiplier = dst_link->pmtud_crypto_timeout_multiplier / 2; log_debug(knet_h, KNET_SUB_PMTUD, "Decreasing PMTUd response timeout multiplier to (%u) for host %u link: %u", dst_link->pmtud_crypto_timeout_multiplier, dst_host->host_id, dst_link->link_id); } } else { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get current time: %s", strerror(errno)); } } if ((dst_link->last_recv_mtu != onwire_len) || (ret)) { dst_link->last_bad_mtu = onwire_len; } else { int found_mtu = 0; if (knet_h->sec_block_size) { if ((onwire_len + knet_h->sec_block_size >= max_mtu_len) || ((dst_link->last_bad_mtu) && (dst_link->last_bad_mtu <= (onwire_len + knet_h->sec_block_size)))) { found_mtu = 1; } } else { if ((onwire_len == max_mtu_len) || ((dst_link->last_bad_mtu) && (dst_link->last_bad_mtu == (onwire_len + 1))) || (dst_link->last_bad_mtu == dst_link->last_good_mtu)) { found_mtu = 1; } } if (found_mtu) { /* * account for IP overhead, knet headers and crypto in PMTU calculation */ dst_link->status.mtu = calc_max_data_outlen(knet_h, onwire_len - ipproto_overhead_len); pthread_mutex_unlock(&knet_h->pmtud_mutex); return 0; } dst_link->last_good_mtu = onwire_len; } } if (kernel_mtu) { onwire_len = kernel_mtu; } else { onwire_len = (dst_link->last_good_mtu + dst_link->last_bad_mtu) / 2; } pthread_mutex_unlock(&knet_h->pmtud_mutex); goto restart; } static int _handle_check_pmtud(knet_handle_t knet_h, struct knet_host *dst_host, struct knet_link *dst_link, int force_run) { uint8_t saved_valid_pmtud; unsigned int saved_pmtud; struct timespec clock_now; unsigned long long diff_pmtud, interval; if (clock_gettime(CLOCK_MONOTONIC, &clock_now) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get monotonic clock"); return 0; } if (!force_run) { interval = knet_h->pmtud_interval * 1000000000llu; /* nanoseconds */ timespec_diff(dst_link->pmtud_last, clock_now, &diff_pmtud); if (diff_pmtud < interval) { return dst_link->has_valid_mtu; } } /* - * status.proto_overhead should include all IP/(UDP|SCTP)/knet headers + * status.proto_overhead should include all IP/(UDP)/knet headers * * please note that it is not the same as link->proto_overhead that - * includes only either UDP or SCTP (at the moment) overhead. + * includes only UDP (at the moment) overhead. */ switch (dst_link->dst_addr.ss_family) { case AF_INET6: dst_link->status.proto_overhead = KNET_PMTUD_OVERHEAD_V6 + dst_link->proto_overhead + KNET_HEADER_ALL_SIZE + knet_h->sec_hash_size + knet_h->sec_salt_size; break; case AF_INET: dst_link->status.proto_overhead = KNET_PMTUD_OVERHEAD_V4 + dst_link->proto_overhead + KNET_HEADER_ALL_SIZE + knet_h->sec_hash_size + knet_h->sec_salt_size; break; } saved_pmtud = dst_link->status.mtu; saved_valid_pmtud = dst_link->has_valid_mtu; log_debug(knet_h, KNET_SUB_PMTUD, "Starting PMTUD for host: %u link: %u", dst_host->host_id, dst_link->link_id); errno = 0; if (_handle_check_link_pmtud(knet_h, dst_host, dst_link) < 0) { if (errno == EDEADLK) { log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD for host: %u link: %u has been rescheduled", dst_host->host_id, dst_link->link_id); dst_link->status.mtu = saved_pmtud; dst_link->has_valid_mtu = saved_valid_pmtud; errno = EDEADLK; return dst_link->has_valid_mtu; } dst_link->has_valid_mtu = 0; } else { if (dst_link->status.mtu < calc_min_mtu(knet_h)) { log_info(knet_h, KNET_SUB_PMTUD, "Invalid MTU detected for host: %u link: %u mtu: %u", dst_host->host_id, dst_link->link_id, dst_link->status.mtu); dst_link->has_valid_mtu = 0; } else { dst_link->has_valid_mtu = 1; } if (dst_link->has_valid_mtu) { if ((saved_pmtud) && (saved_pmtud != dst_link->status.mtu)) { log_info(knet_h, KNET_SUB_PMTUD, "PMTUD link change for host: %u link: %u from %u to %u", dst_host->host_id, dst_link->link_id, saved_pmtud, dst_link->status.mtu); } log_debug(knet_h, KNET_SUB_PMTUD, "PMTUD completed for host: %u link: %u current link mtu: %u", dst_host->host_id, dst_link->link_id, dst_link->status.mtu); /* * set pmtud_last, if we can, after we are done with the PMTUd process * because it can take a very long time. */ dst_link->pmtud_last = clock_now; if (!clock_gettime(CLOCK_MONOTONIC, &clock_now)) { dst_link->pmtud_last = clock_now; } } } if (saved_valid_pmtud != dst_link->has_valid_mtu) { _host_dstcache_update_async(knet_h, dst_host); } return dst_link->has_valid_mtu; } void *_handle_pmtud_link_thread(void *data) { knet_handle_t knet_h = (knet_handle_t) data; struct knet_host *dst_host; struct knet_link *dst_link; int link_idx; unsigned int have_mtu; unsigned int lower_mtu; int link_has_mtu; int force_run = 0; set_thread_status(knet_h, KNET_THREAD_PMTUD, KNET_THREAD_STARTED); knet_h->data_mtu = calc_min_mtu(knet_h); while (!shutdown_in_progress(knet_h)) { usleep(knet_h->threads_timer_res); if (pthread_mutex_lock(&knet_h->pmtud_mutex) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock"); continue; } knet_h->pmtud_abort = 0; knet_h->pmtud_running = 1; force_run = knet_h->pmtud_forcerun; knet_h->pmtud_forcerun = 0; pthread_mutex_unlock(&knet_h->pmtud_mutex); if (force_run) { log_debug(knet_h, KNET_SUB_PMTUD, "PMTUd request to rerun has been received"); } if (pthread_rwlock_rdlock(&knet_h->global_rwlock) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get read lock"); continue; } lower_mtu = KNET_PMTUD_SIZE_V4; have_mtu = 0; for (dst_host = knet_h->host_head; dst_host != NULL; dst_host = dst_host->next) { for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) { dst_link = &dst_host->link[link_idx]; if ((dst_link->status.enabled != 1) || (dst_link->status.connected != 1) || (dst_host->link[link_idx].transport == KNET_TRANSPORT_LOOPBACK) || (!dst_link->last_ping_size) || ((dst_link->dynamic == KNET_LINK_DYNIP) && (dst_link->status.dynconnected != 1))) continue; if (!knet_h->manual_mtu) { link_has_mtu = _handle_check_pmtud(knet_h, dst_host, dst_link, force_run); if (errno == EDEADLK) { goto out_unlock; } if (link_has_mtu) { have_mtu = 1; if (dst_link->status.mtu < lower_mtu) { lower_mtu = dst_link->status.mtu; } } } else { link_has_mtu = _calculate_manual_mtu(knet_h, dst_link); if (link_has_mtu) { have_mtu = 1; if (dst_link->status.mtu < lower_mtu) { lower_mtu = dst_link->status.mtu; } } } } } if (have_mtu) { if (knet_h->data_mtu != lower_mtu) { knet_h->data_mtu = lower_mtu; log_info(knet_h, KNET_SUB_PMTUD, "Global data MTU changed to: %u", knet_h->data_mtu); if (knet_h->pmtud_notify_fn) { knet_h->pmtud_notify_fn(knet_h->pmtud_notify_fn_private_data, knet_h->data_mtu); } } } out_unlock: pthread_rwlock_unlock(&knet_h->global_rwlock); if (pthread_mutex_lock(&knet_h->pmtud_mutex) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock"); } else { knet_h->pmtud_running = 0; pthread_mutex_unlock(&knet_h->pmtud_mutex); } } set_thread_status(knet_h, KNET_THREAD_PMTUD, KNET_THREAD_STOPPED); return NULL; } static void send_pmtud_reply(knet_handle_t knet_h, struct knet_link *src_link, struct knet_header *inbuf) { int err = 0, savederrno = 0, stats_err = 0; unsigned char *outbuf = (unsigned char *)inbuf; ssize_t len, outlen; if (knet_h->onwire_ver_remap) { prep_pmtud_reply_v1(knet_h, inbuf, &outlen); } else { switch (inbuf->kh_version) { case 1: prep_pmtud_reply_v1(knet_h, inbuf, &outlen); break; default: log_warn(knet_h, KNET_SUB_PMTUD, "preparing PMTUD reply onwire version %u not supported", inbuf->kh_version); return; break; } } if (knet_h->crypto_in_use_config) { if (crypto_encrypt_and_sign(knet_h, (const unsigned char *)inbuf, outlen, knet_h->recv_from_links_buf_crypt, &outlen) < 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to encrypt PMTUd reply packet"); return; } outbuf = knet_h->recv_from_links_buf_crypt; stats_err = pthread_mutex_lock(&knet_h->handle_stats_mutex); if (stats_err < 0) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock: %s", strerror(stats_err)); return; } knet_h->stats_extra.tx_crypt_pmtu_reply_packets++; pthread_mutex_unlock(&knet_h->handle_stats_mutex); } savederrno = pthread_mutex_lock(&knet_h->tx_mutex); if (savederrno) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get TX mutex lock: %s", strerror(savederrno)); return; } retry: if (src_link->transport_connected) { if (transport_get_connection_oriented(knet_h, src_link->transport) == TRANSPORT_PROTO_NOT_CONNECTION_ORIENTED) { len = sendto(src_link->outsock, outbuf, outlen, MSG_DONTWAIT | MSG_NOSIGNAL, (struct sockaddr *) &src_link->dst_addr, knet_h->knet_transport_fd_tracker[src_link->outsock].sockaddr_len); } else { len = sendto(src_link->outsock, outbuf, outlen, MSG_DONTWAIT | MSG_NOSIGNAL, NULL, 0); } savederrno = errno; if (len != outlen) { err = transport_tx_sock_error(knet_h, src_link->transport, src_link->outsock, KNET_SUB_PMTUD, len, savederrno); stats_err = pthread_mutex_lock(&src_link->link_stats_mutex); if (stats_err < 0) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock: %s", strerror(stats_err)); return; } switch(err) { case KNET_TRANSPORT_SOCK_ERROR_INTERNAL: log_debug(knet_h, KNET_SUB_PMTUD, "Unable to send PMTUd reply (sock: %d) packet (sendto): %d %s. recorded src ip: %s src port: %s dst ip: %s dst port: %s", src_link->outsock, errno, strerror(errno), src_link->status.src_ipaddr, src_link->status.src_port, src_link->status.dst_ipaddr, src_link->status.dst_port); src_link->status.stats.tx_pmtu_errors++; break; case KNET_TRANSPORT_SOCK_ERROR_IGNORE: src_link->status.stats.tx_pmtu_errors++; break; case KNET_TRANSPORT_SOCK_ERROR_RETRY: src_link->status.stats.tx_pmtu_retries++; pthread_mutex_unlock(&src_link->link_stats_mutex); goto retry; break; } pthread_mutex_unlock(&src_link->link_stats_mutex); } } pthread_mutex_unlock(&knet_h->tx_mutex); } void process_pmtud(knet_handle_t knet_h, struct knet_link *src_link, struct knet_header *inbuf) { /* * at the moment we don't need to take any extra * actions when processing a PMTUd packet, except * sending a reply */ send_pmtud_reply(knet_h, src_link, inbuf); } void process_pmtud_reply(knet_handle_t knet_h, struct knet_link *src_link, struct knet_header *inbuf) { if (pthread_mutex_lock(&knet_h->pmtud_mutex) != 0) { log_debug(knet_h, KNET_SUB_PMTUD, "Unable to get mutex lock"); return; } if (knet_h->onwire_ver_remap) { process_pmtud_reply_v1(knet_h, src_link, inbuf); } else { switch (inbuf->kh_version) { case 1: process_pmtud_reply_v1(knet_h, src_link, inbuf); break; default: log_warn(knet_h, KNET_SUB_PMTUD, "preparing PMTUD reply onwire version %u not supported", inbuf->kh_version); goto out_unlock; break; } } pthread_cond_signal(&knet_h->pmtud_cond); out_unlock: pthread_mutex_unlock(&knet_h->pmtud_mutex); } int knet_handle_pmtud_getfreq(knet_handle_t knet_h, unsigned int *interval) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (!interval) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } *interval = knet_h->pmtud_interval; pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } int knet_handle_pmtud_setfreq(knet_handle_t knet_h, unsigned int interval) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if ((!interval) || (interval > 86400)) { errno = EINVAL; return -1; } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } knet_h->pmtud_interval = interval; log_debug(knet_h, KNET_SUB_PMTUD, "PMTUd interval set to: %u seconds", interval); pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } int knet_handle_enable_pmtud_notify(knet_handle_t knet_h, void *pmtud_notify_fn_private_data, void (*pmtud_notify_fn) ( void *private_data, unsigned int data_mtu)) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get write lock: %s", strerror(savederrno)); errno = savederrno; return -1; } knet_h->pmtud_notify_fn_private_data = pmtud_notify_fn_private_data; knet_h->pmtud_notify_fn = pmtud_notify_fn; if (knet_h->pmtud_notify_fn) { log_debug(knet_h, KNET_SUB_PMTUD, "pmtud_notify_fn enabled"); } else { log_debug(knet_h, KNET_SUB_PMTUD, "pmtud_notify_fn disabled"); } pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } int knet_handle_pmtud_set(knet_handle_t knet_h, unsigned int iface_mtu) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (iface_mtu > KNET_PMTUD_SIZE_V4) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } log_info(knet_h, KNET_SUB_PMTUD, "MTU manually set to: %u", iface_mtu); knet_h->manual_mtu = iface_mtu; force_pmtud_run(knet_h, KNET_SUB_PMTUD, 0, 0); pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } int knet_handle_pmtud_get(knet_handle_t knet_h, unsigned int *data_mtu) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (!data_mtu) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_PMTUD, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } *data_mtu = knet_h->data_mtu; pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } diff --git a/libknet/threads_rx.c b/libknet/threads_rx.c index 3b8fc49c..fb1b8d32 100644 --- a/libknet/threads_rx.c +++ b/libknet/threads_rx.c @@ -1,1251 +1,1249 @@ /* * Copyright (C) 2012-2025 Red Hat, Inc. All rights reserved. * * Authors: Fabio M. Di Nitto * Federico Simoncelli * * This software licensed under LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include "compat.h" #include "compress.h" #include "crypto.h" #include "host.h" #include "links.h" #include "links_acl.h" #include "logging.h" #include "transports.h" #include "transport_common.h" #include "threads_common.h" #include "threads_heartbeat.h" #include "threads_pmtud.h" #include "threads_rx.h" #include "netutils.h" #include "onwire_v1.h" /* * RECV */ /* * return 1 if a > b * return -1 if b > a * return 0 if they are equal */ static inline int _timecmp(struct timespec a, struct timespec b) { if (a.tv_sec != b.tv_sec) { if (a.tv_sec > b.tv_sec) { return 1; } else { return -1; } } else { if (a.tv_nsec > b.tv_nsec) { return 1; } else if (a.tv_nsec < b.tv_nsec) { return -1; } else { return 0; } } } /* * calculate use % of defrag buffers per host * and if % is <= knet_h->defrag_bufs_shrink_threshold for the last second, then half the size */ static void _shrink_defrag_buffers(knet_handle_t knet_h) { struct knet_host *host; struct knet_host_defrag_buf *new_bufs = NULL; struct timespec now; unsigned long long time_diff; /* nanoseconds */ uint16_t i, x, in_use_bufs; uint32_t sum; /* * first run. */ if ((knet_h->defrag_bufs_last_run.tv_sec == 0) && (knet_h->defrag_bufs_last_run.tv_nsec == 0)) { clock_gettime(CLOCK_MONOTONIC, &knet_h->defrag_bufs_last_run); return; } clock_gettime(CLOCK_MONOTONIC, &now); timespec_diff(knet_h->defrag_bufs_last_run, now, &time_diff); if (time_diff < (((unsigned long long)knet_h->defrag_bufs_usage_samples_timespan * 1000000000) / knet_h->defrag_bufs_usage_samples)) { return; } /* * record the last run */ memmove(&knet_h->defrag_bufs_last_run, &now, sizeof(struct timespec)); /* * do the real work: */ for (host = knet_h->host_head; host != NULL; host = host->next) { /* * Update buffer usage stats. We do this for all nodes. */ in_use_bufs = 0; for (i = 0; i < host->allocated_defrag_bufs; i++) { if (host->defrag_bufs[i].in_use) { in_use_bufs++; } } /* * record only % */ host->in_use_defrag_buffers[host->in_use_defrag_buffers_index] = (in_use_bufs * 100 / host->allocated_defrag_bufs); host->in_use_defrag_buffers_index++; /* * make sure to stay within buffer */ if (host->in_use_defrag_buffers_index == knet_h->defrag_bufs_usage_samples) { host->in_use_defrag_buffers_index = 0; } /* * only allow shrinking if we have enough samples */ if (host->in_use_defrag_buffers_samples < knet_h->defrag_bufs_usage_samples) { host->in_use_defrag_buffers_samples++; continue; } /* * only allow shrinking if in use bufs are <= knet_h->defrag_bufs_shrink_threshold% */ if (knet_h->defrag_bufs_reclaim_policy == RECLAIM_POLICY_AVERAGE) { sum = 0; for (i = 0; i < knet_h->defrag_bufs_usage_samples; i++) { sum += host->in_use_defrag_buffers[i]; } sum = sum / knet_h->defrag_bufs_usage_samples; if (sum > knet_h->defrag_bufs_shrink_threshold) { continue; } } else { sum = 0; for (i = 0; i < knet_h->defrag_bufs_usage_samples; i++) { if (host->in_use_defrag_buffers[i] > knet_h->defrag_bufs_shrink_threshold) { sum = 1; } } if (sum) { continue; } } /* * only allow shrinking if allocated bufs > min_defrag_bufs */ if (host->allocated_defrag_bufs == knet_h->defrag_bufs_min) { continue; } /* * compat all the in_use buffers at the beginning. * we the checks above, we are 100% sure they fit */ x = 0; for (i = 0; i < host->allocated_defrag_bufs; i++) { if (host->defrag_bufs[i].in_use) { memmove(&host->defrag_bufs[x], &host->defrag_bufs[i], sizeof(struct knet_host_defrag_buf)); x++; } } /* * memory allocation is not critical. it just means the system is under * memory pressure and we will need to wait our turn to free memory... how odd :) */ new_bufs = realloc(host->defrag_bufs, sizeof(struct knet_host_defrag_buf) * (host->allocated_defrag_bufs / 2)); if (!new_bufs) { log_err(knet_h, KNET_SUB_RX, "Unable to decrease defrag buffers for host %u: %s", host->host_id, strerror(errno)); continue; } host->defrag_bufs = new_bufs; host->allocated_defrag_bufs = host->allocated_defrag_bufs / 2; /* * clear buffer use stats. Old ones are no good for new one */ _clear_defrag_bufs_stats(host); log_debug(knet_h, KNET_SUB_RX, "Defrag buffers for host %u decreased from %u to: %u", host->host_id, host->allocated_defrag_bufs * 2, host->allocated_defrag_bufs); } } /* * check if we can double the defrag buffers. * * return 0 if we cannot reallocate * return 1 if we have more buffers */ static int _realloc_defrag_buffers(knet_handle_t knet_h, struct knet_host *src_host) { struct knet_host_defrag_buf *new_bufs = NULL; int i; /* * max_defrag_bufs is a power of 2 * allocated_defrag_bufs doubles on each iteration. * Sooner or later (and hopefully never) allocated with be == to max. */ if (src_host->allocated_defrag_bufs < knet_h->defrag_bufs_max) { new_bufs = realloc(src_host->defrag_bufs, src_host->allocated_defrag_bufs * 2 * sizeof(struct knet_host_defrag_buf)); if (!new_bufs) { log_err(knet_h, KNET_SUB_RX, "Unable to increase defrag buffers for host %u: %s", src_host->host_id, strerror(errno)); return 0; } /* * keep the math simple here between arrays, pointers and what not. * Init each buffer individually. */ for (i = src_host->allocated_defrag_bufs; i < src_host->allocated_defrag_bufs * 2; i++) { memset(&new_bufs[i], 0, sizeof(struct knet_host_defrag_buf)); } src_host->allocated_defrag_bufs = src_host->allocated_defrag_bufs * 2; src_host->defrag_bufs = new_bufs; /* * clear buffer use stats. Old ones are no good for new one */ _clear_defrag_bufs_stats(src_host); log_debug(knet_h, KNET_SUB_RX, "Defrag buffers for host %u increased from %u to: %u", src_host->host_id, src_host->allocated_defrag_bufs / 2, src_host->allocated_defrag_bufs); return 1; } return 0; } /* * this functions needs to return an index * to a knet_host_defrag_buf. (-1 on errors) */ static int _find_pckt_defrag_buf(knet_handle_t knet_h, struct knet_host *src_host, seq_num_t seq_num) { int i, oldest; uint16_t cur_allocated_defrag_bufs = src_host->allocated_defrag_bufs; /* * check if there is a buffer already in use handling the same seq_num */ for (i = 0; i < src_host->allocated_defrag_bufs; i++) { if (src_host->defrag_bufs[i].in_use) { if (src_host->defrag_bufs[i].pckt_seq == seq_num) { return i; } } } /* * If there is no buffer that's handling the current seq_num * either it's new or it's been reclaimed already. * check if it's been reclaimed/seen before using the defrag circular * buffer. If the pckt has been seen before, the buffer expired (ETIME) * and there is no point to try to defrag it again. */ if (!_seq_num_lookup(knet_h, src_host, seq_num, 1, 0)) { errno = ETIME; return -1; } /* * register the pckt as seen */ _seq_num_set(src_host, seq_num, 1); /* * see if there is a free buffer */ for (i = 0; i < src_host->allocated_defrag_bufs; i++) { if (!src_host->defrag_bufs[i].in_use) { return i; } } /* * check if we can increase num of buffers */ if (_realloc_defrag_buffers(knet_h, src_host)) { return cur_allocated_defrag_bufs + 1; } /* * at this point, there are no free buffers, the pckt is new * and we need to reclaim a buffer, and we will take the one * with the oldest timestamp. It's as good as any. */ oldest = 0; for (i = 0; i < src_host->allocated_defrag_bufs; i++) { if (_timecmp(src_host->defrag_bufs[i].last_update, src_host->defrag_bufs[oldest].last_update) < 0) { oldest = i; } } src_host->defrag_bufs[oldest].in_use = 0; return oldest; } static int _pckt_defrag(knet_handle_t knet_h, struct knet_host *src_host, seq_num_t seq_num, unsigned char *data, ssize_t *len, uint8_t frags, uint8_t frag_seq) { struct knet_host_defrag_buf *defrag_buf; int defrag_buf_idx; defrag_buf_idx = _find_pckt_defrag_buf(knet_h, src_host, seq_num); if (defrag_buf_idx < 0) { return 1; } defrag_buf = &src_host->defrag_bufs[defrag_buf_idx]; /* * if the buf is not is use, then make sure it's clean */ if (!defrag_buf->in_use) { memset(defrag_buf, 0, sizeof(struct knet_host_defrag_buf)); defrag_buf->in_use = 1; defrag_buf->pckt_seq = seq_num; } /* * update timestamp on the buffer */ clock_gettime(CLOCK_MONOTONIC, &defrag_buf->last_update); /* * check if we already received this fragment */ if (defrag_buf->frag_map[frag_seq]) { /* * if we have received this fragment and we didn't clear the buffer * it means that we don't have all fragments yet */ return 1; } /* * we need to handle the last packet with gloves due to its different size */ if (frag_seq == frags) { defrag_buf->last_frag_size = *len; /* * in the event when the last packet arrives first, * we still don't know the offset vs the other fragments (based on MTU), * so we store the fragment at the end of the buffer where it's safe * and take a copy of the len so that we can restore its offset later. * remember we can't use the local MTU for this calculation because pMTU * can be asymettric between the same hosts. */ if (!defrag_buf->frag_size) { defrag_buf->last_first = 1; memmove(defrag_buf->buf + (KNET_MAX_PACKET_SIZE - *len), data, *len); } } else { defrag_buf->frag_size = *len; } if (defrag_buf->frag_size) { memmove(defrag_buf->buf + ((frag_seq - 1) * defrag_buf->frag_size), data, *len); } defrag_buf->frag_recv++; defrag_buf->frag_map[frag_seq] = 1; /* * check if we received all the fragments */ if (defrag_buf->frag_recv == frags) { /* * special case the last pckt */ if (defrag_buf->last_first) { memmove(defrag_buf->buf + ((frags - 1) * defrag_buf->frag_size), defrag_buf->buf + (KNET_MAX_PACKET_SIZE - defrag_buf->last_frag_size), defrag_buf->last_frag_size); } /* * recalculate packet lenght */ *len = ((frags - 1) * defrag_buf->frag_size) + defrag_buf->last_frag_size; /* * copy the pckt back in the user data */ memmove(data, defrag_buf->buf, *len); /* * free this buffer */ defrag_buf->in_use = 0; return 0; } return 1; } static int _handle_data_stats(knet_handle_t knet_h, struct knet_link *src_link, ssize_t len, uint64_t decrypt_time) { int stats_err; /* data stats at the top for consistency with TX */ src_link->status.stats.rx_data_packets++; src_link->status.stats.rx_data_bytes += len; if (decrypt_time) { stats_err = pthread_mutex_lock(&knet_h->handle_stats_mutex); if (stats_err < 0) { log_err(knet_h, KNET_SUB_RX, "Unable to get mutex lock: %s", strerror(stats_err)); return -1; } /* Only update the crypto overhead for data packets. Mainly to be consistent with TX */ if (decrypt_time < knet_h->stats.rx_crypt_time_min) { knet_h->stats.rx_crypt_time_min = decrypt_time; } if (decrypt_time > knet_h->stats.rx_crypt_time_max) { knet_h->stats.rx_crypt_time_max = decrypt_time; } knet_h->stats.rx_crypt_time_ave = (knet_h->stats.rx_crypt_time_ave * knet_h->stats.rx_crypt_packets + decrypt_time) / (knet_h->stats.rx_crypt_packets+1); knet_h->stats.rx_crypt_packets++; pthread_mutex_unlock(&knet_h->handle_stats_mutex); } return 0; } static int _decompress_data(knet_handle_t knet_h, uint8_t decompress_type, unsigned char *data, ssize_t *len, ssize_t header_size) { int err = 0, stats_err = 0; if (decompress_type) { ssize_t decmp_outlen = KNET_DATABUFSIZE_COMPRESS; struct timespec start_time; struct timespec end_time; uint64_t decompress_time; clock_gettime(CLOCK_MONOTONIC, &start_time); err = decompress(knet_h, decompress_type, data, *len - header_size, knet_h->recv_from_links_buf_decompress, &decmp_outlen); clock_gettime(CLOCK_MONOTONIC, &end_time); timespec_diff(start_time, end_time, &decompress_time); stats_err = pthread_mutex_lock(&knet_h->handle_stats_mutex); if (stats_err < 0) { log_err(knet_h, KNET_SUB_RX, "Unable to get mutex lock: %s", strerror(stats_err)); return -1; } if (!err) { /* Collect stats */ if (decompress_time < knet_h->stats.rx_compress_time_min) { knet_h->stats.rx_compress_time_min = decompress_time; } if (decompress_time > knet_h->stats.rx_compress_time_max) { knet_h->stats.rx_compress_time_max = decompress_time; } knet_h->stats.rx_compress_time_ave = (knet_h->stats.rx_compress_time_ave * knet_h->stats.rx_compressed_packets + decompress_time) / (knet_h->stats.rx_compressed_packets+1); knet_h->stats.rx_compressed_packets++; knet_h->stats.rx_compressed_original_bytes += decmp_outlen; knet_h->stats.rx_compressed_size_bytes += *len - KNET_HEADER_SIZE; memmove(data, knet_h->recv_from_links_buf_decompress, decmp_outlen); *len = decmp_outlen + header_size; } else { knet_h->stats.rx_failed_to_decompress++; pthread_mutex_unlock(&knet_h->handle_stats_mutex); log_err(knet_h, KNET_SUB_COMPRESS, "Unable to decompress packet (%d): %s", err, strerror(errno)); return -1; } pthread_mutex_unlock(&knet_h->handle_stats_mutex); } return 0; } static int _check_destination(knet_handle_t knet_h, struct knet_header *inbuf, unsigned char *data, ssize_t len, ssize_t header_size, int8_t *channel) { knet_node_id_t dst_host_ids[KNET_MAX_HOST]; size_t dst_host_ids_entries = 0; int bcast = 1; size_t host_idx; int found = 0; if (knet_h->dst_host_filter_fn) { bcast = knet_h->dst_host_filter_fn( knet_h->dst_host_filter_fn_private_data, data, len - header_size, KNET_NOTIFY_RX, knet_h->host_id, inbuf->kh_node, channel, dst_host_ids, &dst_host_ids_entries); if (bcast < 0) { log_debug(knet_h, KNET_SUB_RX, "Error from dst_host_filter_fn: %d", bcast); return -1; } if ((!bcast) && (!dst_host_ids_entries)) { log_debug(knet_h, KNET_SUB_RX, "Message is unicast but no dst_host_ids_entries"); return -1; } /* check if we are dst for this packet */ if (!bcast) { if (dst_host_ids_entries > KNET_MAX_HOST) { log_debug(knet_h, KNET_SUB_RX, "dst_host_filter_fn returned too many destinations"); return -1; } for (host_idx = 0; host_idx < dst_host_ids_entries; host_idx++) { if (dst_host_ids[host_idx] == knet_h->host_id) { found = 1; break; } } if (!found) { log_debug(knet_h, KNET_SUB_RX, "Packet is not for us"); return -1; } } } return 0; } static int _deliver_data(knet_handle_t knet_h, unsigned char *data, ssize_t len, ssize_t header_size, int8_t channel, struct knet_host *src_host, struct knet_link *src_link) { struct iovec iov_out[2]; ssize_t outlen = 0; uint32_t cur_iov = 0; struct knet_datafd_header datafd_hdr; memset(iov_out, 0, sizeof(iov_out)); if (knet_h->sockfd[channel].flags & KNET_DATAFD_FLAG_RX_RETURN_INFO) { datafd_hdr.size = sizeof(datafd_hdr); datafd_hdr.src_nodeid = src_host->host_id; iov_out[0].iov_base = &datafd_hdr; iov_out[0].iov_len = sizeof(datafd_hdr); cur_iov++; } iov_out[cur_iov].iov_base = (void *) data + outlen; iov_out[cur_iov].iov_len = len - (outlen + header_size); outlen = writev_all(knet_h, knet_h->sockfd[channel].sockfd[knet_h->sockfd[channel].is_created], iov_out, cur_iov+1, src_link, KNET_SUB_RX); if (outlen <= 0) { knet_h->sock_notify_fn(knet_h->sock_notify_fn_private_data, knet_h->sockfd[channel].sockfd[0], channel, KNET_NOTIFY_RX, outlen, errno); return -1; } if ((size_t)outlen != iov_out[0].iov_len) { return -1; } return 0; } static int _fast_data_up(knet_handle_t knet_h, struct knet_host *src_host, struct knet_link *src_link) { if (src_link->received_pong) { log_debug(knet_h, KNET_SUB_RX, "host: %u link: %u received data during valid ping/pong activity. Force link up.", src_host->host_id, src_link->link_id); _link_updown(knet_h, src_host->host_id, src_link->link_id, src_link->status.enabled, 1, 0); return 1; } // host is not eligible for fast data up return 0; } static void _process_data(knet_handle_t knet_h, struct knet_host *src_host, struct knet_link *src_link, struct knet_header *inbuf, ssize_t len, uint64_t decrypt_time) { int8_t channel; uint8_t decompress_type = 0; ssize_t header_size; seq_num_t seq_num; uint8_t frags, frag_seq; unsigned char *data; if (_handle_data_stats(knet_h, src_link, len, decrypt_time) < 0) { return; } /* * register host is sending data. Required to determine if we need * to reset circular buffers. (see onwire_v1.c) */ src_host->got_data = 1; if (knet_h->onwire_ver_remap) { get_data_header_info_v1(knet_h, inbuf, &header_size, &channel, &seq_num, &decompress_type, &frags, &frag_seq); data = get_data_v1(knet_h, inbuf); } else { switch (inbuf->kh_version) { case 1: get_data_header_info_v1(knet_h, inbuf, &header_size, &channel, &seq_num, &decompress_type, &frags, &frag_seq); data = get_data_v1(knet_h, inbuf); break; default: log_warn(knet_h, KNET_SUB_RX, "processing data onwire version %u not supported", inbuf->kh_version); return; break; } } if (!_seq_num_lookup(knet_h, src_host, seq_num, 0, 0)) { if (src_host->link_handler_policy != KNET_LINK_POLICY_ACTIVE) { log_debug(knet_h, KNET_SUB_RX, "Packet has already been delivered"); } return; } if (frags > 1) { /* * len as received from the socket also includes extra stuff * that the defrag code doesn't care about. So strip it * here and readd only for repadding once we are done * defragging * * the defrag code assumes that data packets have all the same size * except the last one that might be smaller. * */ len = len - header_size; if (_pckt_defrag(knet_h, src_host, seq_num, data, &len, frags, frag_seq)) { return; } len = len + header_size; } if (_decompress_data(knet_h, decompress_type, data, &len, header_size) < 0) { return; } if (!src_host->status.reachable) { if (!_fast_data_up(knet_h, src_host, src_link)) { log_debug(knet_h, KNET_SUB_RX, "Source host %u not reachable yet. Discarding packet.", src_host->host_id); return; } } if (knet_h->enabled != 1) /* data forward is disabled */ return; if (_check_destination(knet_h, inbuf, data, len, header_size, &channel) < 0) { return; } if (!knet_h->sockfd[channel].in_use) { log_debug(knet_h, KNET_SUB_RX, "received packet for channel %d but there is no local sock connected", channel); return; } #ifdef ONWIRE_V1_EXTRA_DEBUG if (inbuf->khp_data_v1_checksum != compute_chksum(data, len - header_size)) { log_err(knet_h, KNET_SUB_RX, "Received incorrect data checksum after reassembly from host: %u seq: %u", src_host->host_id, seq_num); /* * give a chance to the log threads to pick up the message */ sleep(1); abort(); } #endif if (_deliver_data(knet_h, data, len, header_size, channel, src_host, src_link) < 0) { return; } _seq_num_set(src_host, seq_num, 0); } static struct knet_header *_decrypt_packet(knet_handle_t knet_h, struct knet_header *inbuf, ssize_t *len, uint64_t *decrypt_time) { int try_decrypt = 0; int i = 0; struct timespec start_time; struct timespec end_time; ssize_t outlen; for (i = 1; i <= KNET_MAX_CRYPTO_INSTANCES; i++) { if (knet_h->crypto_instance[i]) { try_decrypt = 1; break; } } if ((!try_decrypt) && (knet_h->crypto_only == KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC)) { log_debug(knet_h, KNET_SUB_RX, "RX thread configured to accept only crypto packets, but no crypto configs are configured!"); return NULL; } if (try_decrypt) { clock_gettime(CLOCK_MONOTONIC, &start_time); if (crypto_authenticate_and_decrypt(knet_h, (unsigned char *)inbuf, *len, knet_h->recv_from_links_buf_decrypt, &outlen) < 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to decrypt/auth packet"); if (knet_h->crypto_only == KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC) { return NULL; } log_debug(knet_h, KNET_SUB_RX, "Attempting to process packet as clear data"); } else { clock_gettime(CLOCK_MONOTONIC, &end_time); timespec_diff(start_time, end_time, decrypt_time); *len = outlen; inbuf = (struct knet_header *)knet_h->recv_from_links_buf_decrypt; } } return inbuf; } static int _packet_checks(knet_handle_t knet_h, struct knet_header *inbuf, ssize_t len) { #ifdef ONWIRE_V1_EXTRA_DEBUG uint32_t rx_packet_checksum, expected_packet_checksum; #endif if (len < (ssize_t)(KNET_HEADER_SIZE + 1)) { log_debug(knet_h, KNET_SUB_RX, "Packet is too short: %ld", (long)len); return -1; } #ifdef ONWIRE_V1_EXTRA_DEBUG inbuf->kh_node = htons(inbuf->kh_node); rx_packet_checksum = inbuf->kh_checksum; inbuf->kh_checksum = 0; expected_packet_checksum = compute_chksum((const unsigned char *)inbuf, len); if (rx_packet_checksum != expected_packet_checksum) { log_err(knet_h, KNET_SUB_RX, "Received packet with incorrect checksum. Received: %u Expected: %u", rx_packet_checksum, expected_packet_checksum); /* * give a chance to the log threads to pick up the message */ sleep(1); abort(); } inbuf->kh_node = ntohs(inbuf->kh_node); #endif /* * old versions of knet did not advertise max_ver and max_ver is set to 0. */ if (!inbuf->kh_max_ver) { inbuf->kh_max_ver = 1; } /* * if the node joining max version is lower than the min version * then we reject the node */ if (inbuf->kh_max_ver < knet_h->onwire_min_ver) { log_warn(knet_h, KNET_SUB_RX, "Received packet version %u from node %u, lower than currently minimal supported onwire version. Rejecting.", inbuf->kh_version, inbuf->kh_node); return -1; } /* * if the node joining with version higher than our max version * then we reject the node */ if (inbuf->kh_version > knet_h->onwire_max_ver) { log_warn(knet_h, KNET_SUB_RX, "Received packet version %u from node %u, higher than currently maximum supported onwire version. Rejecting.", inbuf->kh_version, inbuf->kh_node); return -1; } /* * if the node joining with version lower than the current in use version * then we reject the node * * NOTE: should we make this configurable and support downgrades? */ if ((!knet_h->onwire_force_ver) && (inbuf->kh_version < knet_h->onwire_ver) && (inbuf->kh_max_ver > inbuf->kh_version)) { log_warn(knet_h, KNET_SUB_RX, "Received packet version %u from node %u, lower than currently in use onwire version. Rejecting.", inbuf->kh_version, inbuf->kh_node); return -1; } return 0; } static void _handle_dynip(knet_handle_t knet_h, struct knet_host *src_host, struct knet_link *src_link, int sockfd, const struct knet_mmsghdr *msg) { if (src_link->dynamic == KNET_LINK_DYNIP) { if (cmpaddr(&src_link->dst_addr, msg->msg_hdr.msg_name) != 0) { log_debug(knet_h, KNET_SUB_RX, "host: %u link: %u appears to have changed ip address", src_host->host_id, src_link->link_id); memmove(&src_link->dst_addr, msg->msg_hdr.msg_name, sizeof(struct sockaddr_storage)); if (knet_addrtostr(&src_link->dst_addr, sockaddr_len(&src_link->dst_addr), src_link->status.dst_ipaddr, KNET_MAX_HOST_LEN, src_link->status.dst_port, KNET_MAX_PORT_LEN) != 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to resolve ???"); snprintf(src_link->status.dst_ipaddr, KNET_MAX_HOST_LEN - 1, "Unknown!!!"); snprintf(src_link->status.dst_port, KNET_MAX_PORT_LEN - 1, "??"); } else { log_info(knet_h, KNET_SUB_RX, "host: %u link: %u new connection established from: %s:%s", src_host->host_id, src_link->link_id, src_link->status.dst_ipaddr, src_link->status.dst_port); } } /* * transport has already accepted the connection here * otherwise we would not be receiving packets */ transport_link_dyn_connect(knet_h, sockfd, src_link); } } /* * processing incoming packets vs access lists */ static int _check_rx_acl(knet_handle_t knet_h, struct knet_link *src_link, const struct knet_mmsghdr *msg) { if (knet_h->use_access_lists) { if (!check_validate(knet_h, src_link, msg->msg_hdr.msg_name)) { char src_ipaddr[KNET_MAX_HOST_LEN]; char src_port[KNET_MAX_PORT_LEN]; memset(src_ipaddr, 0, KNET_MAX_HOST_LEN); memset(src_port, 0, KNET_MAX_PORT_LEN); if (knet_addrtostr(msg->msg_hdr.msg_name, sockaddr_len(msg->msg_hdr.msg_name), src_ipaddr, KNET_MAX_HOST_LEN, src_port, KNET_MAX_PORT_LEN) < 0) { log_warn(knet_h, KNET_SUB_RX, "Packet rejected: unable to resolve host/port"); } else { log_warn(knet_h, KNET_SUB_RX, "Packet rejected from %s:%s", src_ipaddr, src_port); } return 0; } } return 1; } static void _parse_recv_from_links(knet_handle_t knet_h, int sockfd, const struct knet_mmsghdr *msg) { int savederrno = 0, stats_err = 0; struct knet_host *src_host; struct knet_link *src_link; uint64_t decrypt_time = 0; struct knet_header *inbuf = msg->msg_hdr.msg_iov->iov_base; ssize_t len = msg->msg_len; int i, found_link = 0; inbuf = _decrypt_packet(knet_h, inbuf, &len, &decrypt_time); if (!inbuf) { char src_ipaddr[KNET_MAX_HOST_LEN]; char src_port[KNET_MAX_PORT_LEN]; memset(src_ipaddr, 0, KNET_MAX_HOST_LEN); memset(src_port, 0, KNET_MAX_PORT_LEN); if (knet_addrtostr(msg->msg_hdr.msg_name, sockaddr_len(msg->msg_hdr.msg_name), src_ipaddr, KNET_MAX_HOST_LEN, src_port, KNET_MAX_PORT_LEN) < 0) { log_err(knet_h, KNET_SUB_RX, "Unable to decrypt packet from unknown host/port (size %zu)!", len); } else { log_err(knet_h, KNET_SUB_RX, "Unable to decrypt packet from %s:%s (size %zu)!", src_ipaddr, src_port, len); } return; } inbuf->kh_node = ntohs(inbuf->kh_node); if (_packet_checks(knet_h, inbuf, len) < 0) { if (knet_h->rx_odd_packets < KNET_RX_ODD_PACKETS_THRESHOLD) { knet_h->rx_odd_packets++; } else { log_warn(knet_h, KNET_SUB_RX, "This node has received more than %u packets that have failed basic sanity checks", KNET_RX_ODD_PACKETS_THRESHOLD); log_warn(knet_h, KNET_SUB_RX, "It is highly recommended to check if all nodes are using the same crypto configuration"); knet_h->rx_odd_packets = 0; } return; } /* * determine source host */ src_host = knet_h->host_index[inbuf->kh_node]; if (src_host == NULL) { /* host not found */ log_debug(knet_h, KNET_SUB_RX, "Unable to find source host for this packet"); return; } /* * deteremine source link */ if (inbuf->kh_type == KNET_HEADER_TYPE_PING) { _handle_onwire_version(knet_h, src_host, inbuf); if (knet_h->onwire_ver_remap) { src_link = get_link_from_pong_v1(knet_h, src_host, inbuf); } else { switch (inbuf->kh_version) { case 1: src_link = get_link_from_pong_v1(knet_h, src_host, inbuf); break; default: log_warn(knet_h, KNET_SUB_RX, "Parsing ping onwire version %u not supported", inbuf->kh_version); return; break; } } if (!_check_rx_acl(knet_h, src_link, msg)) { return; } _handle_dynip(knet_h, src_host, src_link, sockfd, msg); } else { /* all other packets */ for (i = 0; i < KNET_MAX_LINK; i++) { src_link = &src_host->link[i]; if (cmpaddr(&src_link->dst_addr, msg->msg_hdr.msg_name) == 0) { found_link = 1; break; } } if (found_link) { /* * this check is currently redundant.. Keep it here for now */ if (!_check_rx_acl(knet_h, src_link, msg)) { return; } } else { log_debug(knet_h, KNET_SUB_RX, "Unable to determine source link for data packet. Discarding packet."); return; } } stats_err = pthread_mutex_lock(&src_link->link_stats_mutex); if (stats_err) { log_err(knet_h, KNET_SUB_RX, "Unable to get stats mutex lock for host %u link %u: %s", src_host->host_id, src_link->link_id, strerror(savederrno)); return; } switch (inbuf->kh_type) { case KNET_HEADER_TYPE_DATA: _process_data(knet_h, src_host, src_link, inbuf, len, decrypt_time); break; case KNET_HEADER_TYPE_PING: process_ping(knet_h, src_host, src_link, inbuf, len); break; case KNET_HEADER_TYPE_PONG: process_pong(knet_h, src_host, src_link, inbuf, len); break; case KNET_HEADER_TYPE_PMTUD: src_link->status.stats.rx_pmtu_packets++; src_link->status.stats.rx_pmtu_bytes += len; /* Unlock so we don't deadlock with tx_mutex */ pthread_mutex_unlock(&src_link->link_stats_mutex); process_pmtud(knet_h, src_link, inbuf); return; /* Don't need to unlock link_stats_mutex */ break; case KNET_HEADER_TYPE_PMTUD_REPLY: src_link->status.stats.rx_pmtu_packets++; src_link->status.stats.rx_pmtu_bytes += len; /* pmtud_mutex can't be acquired while we hold a link_stats_mutex (ordering) */ pthread_mutex_unlock(&src_link->link_stats_mutex); process_pmtud_reply(knet_h, src_link, inbuf); return; break; default: pthread_mutex_unlock(&src_link->link_stats_mutex); return; break; } pthread_mutex_unlock(&src_link->link_stats_mutex); } static void _handle_recv_from_links(knet_handle_t knet_h, int sockfd, struct knet_mmsghdr *msg) { int err, savederrno; int i, msg_recv, transport; if (pthread_rwlock_rdlock(&knet_h->global_rwlock) != 0) { log_debug(knet_h, KNET_SUB_RX, "Unable to get global read lock"); return; } if (_is_valid_fd(knet_h, sockfd) < 1) { /* * this is normal if a fd got an event and before we grab the read lock * and the link is removed by another thread */ goto exit_unlock; } transport = knet_h->knet_transport_fd_tracker[sockfd].transport; /* * reset msg_namelen to buffer size because after recvmmsg * each msg_namelen will contain sizeof sockaddr_in or sockaddr_in6 */ for (i = 0; i < PCKT_RX_BUFS; i++) { msg[i].msg_hdr.msg_namelen = knet_h->knet_transport_fd_tracker[sockfd].sockaddr_len; } msg_recv = _recvmmsg(sockfd, &msg[0], PCKT_RX_BUFS, MSG_DONTWAIT | MSG_NOSIGNAL); savederrno = errno; /* * WARNING: man page for recvmmsg is wrong. Kernel implementation here: * recvmmsg can return: * -1 on error * 0 if the previous run of recvmmsg recorded an error on the socket * N number of messages (see exception below). * * If there is an error from recvmsg after receiving a frame or more, the recvmmsg * loop is interrupted, error recorded in the socket (getsockopt(SO_ERROR) and * it will be visibile in the next run. * * Need to be careful how we handle errors at this stage. * * error messages need to be handled on a per transport/protocol base * at this point we have different layers of error handling * - msg_recv < 0 -> error from this run * msg_recv = 0 -> error from previous run and error on socket needs to be cleared * - per-transport message data - * example: msg[i].msg_hdr.msg_flags & MSG_NOTIFICATION or msg_len for SCTP == EOF, + * example: msg[i].msg_hdr.msg_flags & MSG_NOTIFICATION, * but for UDP it is perfectly legal to receive a 0 bytes message.. go figure - * - NOTE: on SCTP MSG_NOTIFICATION we get msg_recv == PCKT_FRAG_MAX messages and no - * errno set. That means the error api needs to be able to abort the loop below. */ if (msg_recv <= 0) { transport_rx_sock_error(knet_h, transport, sockfd, msg_recv, savederrno); goto exit_unlock; } for (i = 0; i < msg_recv; i++) { err = transport_rx_is_data(knet_h, transport, sockfd, &msg[i]); /* * TODO: make this section silent once we are confident * all protocols packet handlers are good */ switch(err) { case KNET_TRANSPORT_RX_ISDATA_ERROR: /* on error */ log_debug(knet_h, KNET_SUB_RX, "Transport reported error parsing packet"); goto exit_unlock; break; case KNET_TRANSPORT_RX_NOT_DATA_CONTINUE: /* packet is not data and we should continue the packet process loop */ log_debug(knet_h, KNET_SUB_RX, "Transport reported no data, continue"); break; case KNET_TRANSPORT_RX_NOT_DATA_STOP: /* packet is not data and we should STOP the packet process loop */ log_debug(knet_h, KNET_SUB_RX, "Transport reported no data, stop"); goto exit_unlock; break; case KNET_TRANSPORT_RX_IS_DATA: /* packet is data and should be parsed as such */ _parse_recv_from_links(knet_h, sockfd, &msg[i]); break; case KNET_TRANSPORT_RX_OOB_DATA_CONTINUE: log_debug(knet_h, KNET_SUB_RX, "Transport is processing sock OOB data, continue"); break; case KNET_TRANSPORT_RX_OOB_DATA_STOP: log_debug(knet_h, KNET_SUB_RX, "Transport has completed processing sock OOB data, stop"); goto exit_unlock; break; } } exit_unlock: _shrink_defrag_buffers(knet_h); pthread_rwlock_unlock(&knet_h->global_rwlock); } void *_handle_recv_from_links_thread(void *data) { int i, nev; knet_handle_t knet_h = (knet_handle_t) data; struct epoll_event events[KNET_EPOLL_MAX_EVENTS]; struct sockaddr_storage address[PCKT_RX_BUFS]; struct knet_mmsghdr msg[PCKT_RX_BUFS]; struct iovec iov_in[PCKT_RX_BUFS]; #if defined(IP_PKTINFO) || defined(IPV6_PKTINFO) unsigned char control_in[PCKT_RX_BUFS][CMSG_SPACE(sizeof(struct in6_pktinfo))]; #endif set_thread_status(knet_h, KNET_THREAD_RX, KNET_THREAD_STARTED); memset(&msg, 0, sizeof(msg)); memset(&events, 0, sizeof(events)); for (i = 0; i < PCKT_RX_BUFS; i++) { iov_in[i].iov_base = (void *)knet_h->recv_from_links_buf[i]; iov_in[i].iov_len = KNET_DATABUFSIZE; memset(&msg[i].msg_hdr, 0, sizeof(struct msghdr)); msg[i].msg_hdr.msg_name = &address[i]; msg[i].msg_hdr.msg_namelen = sizeof(struct sockaddr_storage); /* Real value filled in before actual use */ msg[i].msg_hdr.msg_iov = &iov_in[i]; msg[i].msg_hdr.msg_iovlen = 1; #if defined(IP_PKTINFO) || defined(IPV6_PKTINFO) msg[i].msg_hdr.msg_control = &control_in[i][0]; msg[i].msg_hdr.msg_controllen = CMSG_SPACE(sizeof(struct in6_pktinfo)); /* Largest of the two pktinfo structs */ #endif } while (!shutdown_in_progress(knet_h)) { nev = epoll_wait(knet_h->recv_from_links_epollfd, events, KNET_EPOLL_MAX_EVENTS, knet_h->threads_timer_res / 1000); /* * the RX threads only need to notify that there has been at least * one successful run after queue flush has been requested. * See setfwd in handle.c */ if (get_thread_flush_queue(knet_h, KNET_THREAD_RX) == KNET_THREAD_QUEUE_FLUSH) { set_thread_flush_queue(knet_h, KNET_THREAD_RX, KNET_THREAD_QUEUE_FLUSHED); } /* * we use timeout to detect if thread is shutting down */ if (nev == 0) { continue; } for (i = 0; i < nev; i++) { _handle_recv_from_links(knet_h, events[i].data.fd, msg); } } set_thread_status(knet_h, KNET_THREAD_RX, KNET_THREAD_STOPPED); return NULL; } ssize_t knet_recv(knet_handle_t knet_h, char *buff, const size_t buff_len, const int8_t channel) { int savederrno = 0; ssize_t err = 0; struct iovec iov_in; if (!_is_valid_handle(knet_h)) { return -1; } if (buff == NULL) { errno = EINVAL; return -1; } if (buff_len <= 0) { errno = EINVAL; return -1; } if (buff_len > KNET_MAX_PACKET_SIZE) { errno = EINVAL; return -1; } if (channel < 0) { errno = EINVAL; return -1; } if (channel >= KNET_DATAFD_MAX) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } if (!knet_h->sockfd[channel].in_use) { savederrno = EINVAL; err = -1; goto out_unlock; } memset(&iov_in, 0, sizeof(iov_in)); iov_in.iov_base = (void *)buff; iov_in.iov_len = buff_len; err = readv(knet_h->sockfd[channel].sockfd[0], &iov_in, 1); savederrno = errno; out_unlock: pthread_rwlock_unlock(&knet_h->global_rwlock); errno = err ? savederrno : 0; return err; } diff --git a/libknet/transport_sctp.c b/libknet/transport_sctp.c deleted file mode 100644 index b7d54fef..00000000 --- a/libknet/transport_sctp.c +++ /dev/null @@ -1,1637 +0,0 @@ -/* - * Copyright (C) 2016-2025 Red Hat, Inc. All rights reserved. - * - * Author: Christine Caulfield - * - * This software licensed under LGPL-2.0+ - */ - -#include "config.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "compat.h" -#include "host.h" -#include "links.h" -#include "links_acl.h" -#include "links_acl_ip.h" -#include "logging.h" -#include "netutils.h" -#include "common.h" -#include "transport_common.h" -#include "transports.h" -#include "threads_common.h" - -#ifdef HAVE_NETINET_SCTP_H -#include -#include "transport_sctp.h" - -typedef struct sctp_handle_info { - struct qb_list_head listen_links_list; - struct qb_list_head connect_links_list; - int connect_epollfd; - int connectsockfd[2]; - int listen_epollfd; - int listensockfd[2]; - pthread_t connect_thread; - pthread_t listen_thread; - socklen_t event_subscribe_kernel_size; - char *event_subscribe_buffer; -} sctp_handle_info_t; - -/* - * use by fd_tracker data type - */ -#define SCTP_NO_LINK_INFO 0 -#define SCTP_LISTENER_LINK_INFO 1 -#define SCTP_ACCEPTED_LINK_INFO 2 -#define SCTP_CONNECT_LINK_INFO 3 - -/* - * this value is per listener - */ -#define MAX_ACCEPTED_SOCKS 256 - -typedef struct sctp_listen_link_info { - struct qb_list_head list; - int listen_sock; - int accepted_socks[MAX_ACCEPTED_SOCKS]; - struct sockaddr_storage src_address; - int on_listener_epoll; - int on_rx_epoll; - int sock_shutdown; -} sctp_listen_link_info_t; - -typedef struct sctp_accepted_link_info { - char mread_buf[KNET_DATABUFSIZE]; - ssize_t mread_len; - sctp_listen_link_info_t *link_info; -} sctp_accepted_link_info_t ; - -typedef struct sctp_connect_link_info { - struct qb_list_head list; - sctp_listen_link_info_t *listener; - struct knet_link *link; - struct sockaddr_storage dst_address; - int connect_sock; - int on_rx_epoll; - int close_sock; - int sock_shutdown; -} sctp_connect_link_info_t; - -/* - * socket handling functions - * - * those functions do NOT perform locking. locking - * should be handled in the right context from callers - */ - -/* - * sockets are removed from rx_epoll from callers - * see also error handling functions - */ -static int _close_connect_socket(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int err = 0, savederrno = 0; - struct epoll_event ev; - sctp_connect_link_info_t *info = kn_link->transport_link; - - if (info->connect_sock != -1) { - if (info->on_rx_epoll) { - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = info->connect_sock; - if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_DEL, info->connect_sock, &ev)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to remove connected socket from epoll pool: %s", - strerror(savederrno)); - goto exit_error; - } - info->on_rx_epoll = 0; - } - - if (_set_fd_tracker(knet_h, info->connect_sock, KNET_MAX_TRANSPORTS, SCTP_NO_LINK_INFO, 0, NULL, -1) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set fd tracker: %s", - strerror(savederrno)); - } else { - close(info->connect_sock); - info->connect_sock = -1; - } - } - -exit_error: - errno = savederrno; - return err; -} - -static int _enable_sctp_notifications(knet_handle_t knet_h, int sock, const char *type) -{ - int err = 0, savederrno = 0; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - - if (setsockopt(sock, IPPROTO_SCTP, SCTP_EVENTS, - handle_info->event_subscribe_buffer, - handle_info->event_subscribe_kernel_size) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to enable %s events: %s", - type, strerror(savederrno)); - } - - errno = savederrno; - return err; -} - -static int _configure_sctp_socket(knet_handle_t knet_h, int sock, struct sockaddr_storage *address, uint64_t flags, const char *type) -{ - int err = 0, savederrno = 0; - int value; - int level; - -#ifdef SOL_SCTP - level = SOL_SCTP; -#else - level = IPPROTO_SCTP; -#endif - - if (_configure_transport_socket(knet_h, sock, address, flags, type) < 0) { - savederrno = errno; - err = -1; - goto exit_error; - } - - value = 1; - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(value)) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSPORT, "Unable to set reuseaddr on socket %d: %s", - sock, strerror(savederrno)); - goto exit_error; - } - - value = 1; - if (setsockopt(sock, level, SCTP_NODELAY, &value, sizeof(value)) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSPORT, "Unable to set sctp nodelay: %s", - strerror(savederrno)); - goto exit_error; - } - - if (_enable_sctp_notifications(knet_h, sock, type) < 0) { - savederrno = errno; - err = -1; - } - -exit_error: - errno = savederrno; - return err; -} - -static int _reconnect_socket(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int err = 0, savederrno = 0; - sctp_connect_link_info_t *info = kn_link->transport_link; - - if (connect(info->connect_sock, (struct sockaddr *)&kn_link->dst_addr, sockaddr_len(&kn_link->dst_addr)) < 0) { - savederrno = errno; - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP socket %d received error: %s", info->connect_sock, strerror(savederrno)); - if ((savederrno != EALREADY) && (savederrno != EINPROGRESS) && (savederrno != EISCONN)) { - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to connect SCTP socket %d: %s", - info->connect_sock, strerror(savederrno)); - } - } - - errno = savederrno; - return err; -} - -static int _create_connect_socket(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int err = 0, savederrno = 0; - struct epoll_event ev; - sctp_connect_link_info_t *info = kn_link->transport_link; - int connect_sock; - struct sockaddr_storage connect_addr; - - connect_sock = socket(kn_link->dst_addr.ss_family, SOCK_STREAM, IPPROTO_SCTP); - if (connect_sock < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to create send/recv socket: %s", - strerror(savederrno)); - goto exit_error; - } - - if (_configure_sctp_socket(knet_h, connect_sock, &kn_link->dst_addr, kn_link->flags, "SCTP connect") < 0) { - savederrno = errno; - err = -1; - goto exit_error; - } - - memset(&connect_addr, 0, sizeof(struct sockaddr_storage)); - if (knet_strtoaddr(kn_link->status.src_ipaddr, "0", &connect_addr, sockaddr_len(&connect_addr)) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to resolve connecting socket: %s", - strerror(savederrno)); - goto exit_error; - - } - - if (bind(connect_sock, (struct sockaddr *)&connect_addr, sockaddr_len(&connect_addr)) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to bind connecting socket: %s", - strerror(savederrno)); - goto exit_error; - } - - if (_set_fd_tracker(knet_h, connect_sock, KNET_TRANSPORT_SCTP, SCTP_CONNECT_LINK_INFO, sockaddr_len(&kn_link->src_addr), info, -1) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set fd tracker: %s", - strerror(savederrno)); - goto exit_error; - } - - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = connect_sock; - if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_ADD, connect_sock, &ev)) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to add connected socket to epoll pool: %s", - strerror(errno)); - } - info->on_rx_epoll = 1; - - info->connect_sock = connect_sock; - info->close_sock = 0; - kn_link->outsock = info->connect_sock; - - if (_reconnect_socket(knet_h, kn_link) < 0) { - savederrno = errno; - err = -1; - goto exit_error; - } - -exit_error: - if (err) { - if (connect_sock >= 0) { - close(connect_sock); - } - } - errno = savederrno; - return err; -} - -static void _lock_sleep_relock(knet_handle_t knet_h) -{ - int i = 0; - - /* Don't hold onto the lock while sleeping */ - pthread_rwlock_unlock(&knet_h->global_rwlock); - - while (i < 5) { - usleep(knet_h->threads_timer_res / 16); - if (!pthread_rwlock_rdlock(&knet_h->global_rwlock)) { - /* - * lock acquired, we can go out - */ - return; - } else { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to get read lock!"); - i++; - } - } - /* - * time to crash! if we cannot re-acquire the lock - * there is no easy way out of this one - */ - assert(0); -} - -transport_sock_error_t sctp_transport_tx_sock_error(knet_handle_t knet_h, int sockfd, int subsys, int recv_err, int recv_errno) -{ - sctp_connect_link_info_t *connect_info = knet_h->knet_transport_fd_tracker[sockfd].data; - sctp_accepted_link_info_t *accepted_info = knet_h->knet_transport_fd_tracker[sockfd].data; - sctp_listen_link_info_t *listen_info; - - if (recv_err < 0) { - switch (knet_h->knet_transport_fd_tracker[sockfd].data_type) { - case SCTP_CONNECT_LINK_INFO: - if (connect_info->link->transport_connected == 0) { - return KNET_TRANSPORT_SOCK_ERROR_INTERNAL; - } - break; - case SCTP_ACCEPTED_LINK_INFO: - listen_info = accepted_info->link_info; - if (listen_info->listen_sock != sockfd) { - if (listen_info->on_rx_epoll == 0) { - return KNET_TRANSPORT_SOCK_ERROR_INTERNAL; - } - } - break; - } - if (recv_errno == EAGAIN) { - log_trace(knet_h, KNET_SUB_TRANSP_SCTP, "Sock: %d is overloaded. Slowing TX down", sockfd); - _lock_sleep_relock(knet_h); - return KNET_TRANSPORT_SOCK_ERROR_RETRY; - } - return KNET_TRANSPORT_SOCK_ERROR_INTERNAL; - } - return KNET_TRANSPORT_SOCK_ERROR_IGNORE; -} - -/* - * socket error management functions - * - * both called with global read lock. - * - * NOTE: we need to remove the fd from the epoll as soon as possible - * even before we notify the respective thread to take care of it - * because scheduling can make it so that this thread will overload - * and the threads supposed to take care of the error will never - * be able to take action. - * we CANNOT handle FDs here directly (close/reconnect/etc) due - * to locking context. We need to delegate that to their respective - * management threads within the global write lock. - * - * this function is called from: - * - RX thread with recv_err <= 0 directly on recvmmsg error - * - transport_rx_is_data when msg_len == 0 (recv_err = 1) - * - transport_rx_is_data on notification (recv_err = 2) - * - * basically this small abuse of recv_err is to detect notifications - * generated by sockets created by listen(). - */ -transport_sock_error_t sctp_transport_rx_sock_error(knet_handle_t knet_h, int sockfd, int recv_err, int recv_errno) -{ - struct epoll_event ev; - sctp_accepted_link_info_t *accepted_info = knet_h->knet_transport_fd_tracker[sockfd].data; - sctp_listen_link_info_t *listen_info; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - - switch (knet_h->knet_transport_fd_tracker[sockfd].data_type) { - case SCTP_CONNECT_LINK_INFO: - /* - * all connect link have notifications enabled - * and we accept only data from notification and - * generic recvmmsg errors. - * - * Errors generated by msg_len 0 can be ignored because - * they follow a notification (double notification) - */ - if (recv_err != 1) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Notifying connect thread that sockfd %d received an error", sockfd); - if (sendto(handle_info->connectsockfd[1], &sockfd, sizeof(int), MSG_DONTWAIT | MSG_NOSIGNAL, NULL, 0) != sizeof(int)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to notify connect thread: %s", strerror(errno)); - } - } - break; - case SCTP_ACCEPTED_LINK_INFO: - listen_info = accepted_info->link_info; - if (listen_info->listen_sock != sockfd) { - if (recv_err != 1) { - if (listen_info->on_rx_epoll) { - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = sockfd; - if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_DEL, sockfd, &ev)) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to remove EOFed socket from epoll pool: %s", - strerror(errno)); - return -1; - } - listen_info->on_rx_epoll = 0; - } - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Notifying listen thread that sockfd %d received an error", sockfd); - if (sendto(handle_info->listensockfd[1], &sockfd, sizeof(int), MSG_DONTWAIT | MSG_NOSIGNAL, NULL, 0) != sizeof(int)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to notify listen thread: %s", strerror(errno)); - } - } - } else { - /* - * this means the listen() socket has generated - * a notification. now what? :-) - */ - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received stray notification for listen() socket %d", sockfd); - } - break; - default: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received unknown notification? %d", sockfd); - break; - } - /* - * Under RX pressure we need to give time to IPC to pick up the message - */ - - _lock_sleep_relock(knet_h); - return 0; -} - -/* - * NOTE: sctp_transport_rx_is_data is called with global rdlock - * delegate any FD error management to sctp_transport_rx_sock_error - * and keep this code to parsing incoming data only - */ -transport_rx_isdata_t sctp_transport_rx_is_data(knet_handle_t knet_h, int sockfd, struct knet_mmsghdr *msg) -{ - size_t i; - struct iovec *iov = msg->msg_hdr.msg_iov; - size_t iovlen = msg->msg_hdr.msg_iovlen; - struct sctp_assoc_change *sac; - union sctp_notification *snp; - sctp_accepted_link_info_t *listen_info = knet_h->knet_transport_fd_tracker[sockfd].data; - sctp_connect_link_info_t *connect_info = knet_h->knet_transport_fd_tracker[sockfd].data; - - if (!(msg->msg_hdr.msg_flags & MSG_NOTIFICATION)) { - if (msg->msg_len == 0) { - /* - * NOTE: with event notification enabled, we receive error twice: - * 1) from the event notification - * 2) followed by a 0 byte msg_len - * - * the event handler should take care to avoid #2 by stopping - * the rx thread from processing more packets than necessary. - */ - if (knet_h->knet_transport_fd_tracker[sockfd].data_type == SCTP_CONNECT_LINK_INFO) { - if (connect_info->sock_shutdown) { - return KNET_TRANSPORT_RX_OOB_DATA_CONTINUE; - } - } else { - if (listen_info->link_info->sock_shutdown) { - return KNET_TRANSPORT_RX_OOB_DATA_CONTINUE; - } - } - /* - * this is pretty much dead code and we should never hit it. - * keep it for safety and avoid the rx thread to process - * bad info / data. - */ - return KNET_TRANSPORT_RX_NOT_DATA_STOP; - } - /* - * missing MSG_EOR has to be treated as a short read - * from the socket and we need to fill in the mread buf - * while we wait for MSG_EOR - */ - if (!(msg->msg_hdr.msg_flags & MSG_EOR)) { - /* - * copy the incoming data into mread_buf + mread_len (incremental) - * and increase mread_len - */ - memmove(listen_info->mread_buf + listen_info->mread_len, iov->iov_base, msg->msg_len); - listen_info->mread_len = listen_info->mread_len + msg->msg_len; - return KNET_TRANSPORT_RX_NOT_DATA_CONTINUE; - } - /* - * got EOR. - * if mread_len is > 0 we are completing a packet from short reads - * complete reassembling the packet in mread_buf, copy it back in the iov - * and set the iov/msg len numbers (size) correctly - */ - if (listen_info->mread_len) { - /* - * add last fragment to mread_buf - */ - memmove(listen_info->mread_buf + listen_info->mread_len, iov->iov_base, msg->msg_len); - listen_info->mread_len = listen_info->mread_len + msg->msg_len; - /* - * move all back into the iovec - */ - memmove(iov->iov_base, listen_info->mread_buf, listen_info->mread_len); - msg->msg_len = listen_info->mread_len; - listen_info->mread_len = 0; - } - return KNET_TRANSPORT_RX_IS_DATA; - } - - if (!(msg->msg_hdr.msg_flags & MSG_EOR)) { - return KNET_TRANSPORT_RX_NOT_DATA_STOP; - } - - for (i = 0; i < iovlen; i++) { - snp = iov[i].iov_base; - - switch (snp->sn_header.sn_type) { - case SCTP_ASSOC_CHANGE: - sac = &snp->sn_assoc_change; - switch (sac->sac_state) { - case SCTP_COMM_LOST: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp assoc change socket %d: comm_lost", sockfd); - if (knet_h->knet_transport_fd_tracker[sockfd].data_type == SCTP_CONNECT_LINK_INFO) { - connect_info->close_sock = 1; - connect_info->link->transport_connected = 0; - } - sctp_transport_rx_sock_error(knet_h, sockfd, 2, 0); - return KNET_TRANSPORT_RX_OOB_DATA_STOP; - break; - case SCTP_COMM_UP: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp assoc change socket %d: comm_up", sockfd); - if (knet_h->knet_transport_fd_tracker[sockfd].data_type == SCTP_CONNECT_LINK_INFO) { - connect_info->link->transport_connected = 1; - } - break; - case SCTP_RESTART: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp assoc change socket %d: restart", sockfd); - break; - case SCTP_SHUTDOWN_COMP: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp assoc change socket %d: shutdown comp", sockfd); - if (knet_h->knet_transport_fd_tracker[sockfd].data_type == SCTP_CONNECT_LINK_INFO) { - connect_info->close_sock = 1; - } - sctp_transport_rx_sock_error(knet_h, sockfd, 2, 0); - return KNET_TRANSPORT_RX_OOB_DATA_STOP; - break; - case SCTP_CANT_STR_ASSOC: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp assoc change socket %d: cant str assoc", sockfd); - sctp_transport_rx_sock_error(knet_h, sockfd, 2, 0); - break; - default: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp assoc change socket %d: unknown %d", sockfd, sac->sac_state); - break; - } - break; - case SCTP_SHUTDOWN_EVENT: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp shutdown event socket %d", sockfd); - if (knet_h->knet_transport_fd_tracker[sockfd].data_type == SCTP_CONNECT_LINK_INFO) { - connect_info->link->transport_connected = 0; - connect_info->sock_shutdown = 1; - } else { - listen_info->link_info->sock_shutdown = 1; - } - break; - case SCTP_SEND_FAILED: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp send failed socket: %d", sockfd); - break; - case SCTP_PEER_ADDR_CHANGE: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp peer addr change socket %d", sockfd); - break; - case SCTP_REMOTE_ERROR: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] sctp remote error socket %d", sockfd); - break; - default: - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "[event] unknown sctp event socket: %d type: %hu", sockfd, snp->sn_header.sn_type); - break; - } - } - return KNET_TRANSPORT_RX_OOB_DATA_CONTINUE; -} - -int sctp_transport_link_is_down(knet_handle_t knet_h, struct knet_link *kn_link) -{ - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - sctp_connect_link_info_t *info = kn_link->transport_link; - - kn_link->transport_connected = 0; - info->close_sock = 1; - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Notifying connect thread that sockfd %d received a link down event", info->connect_sock); - if (sendto(handle_info->connectsockfd[1], &info->connect_sock, sizeof(int), MSG_DONTWAIT | MSG_NOSIGNAL, NULL, 0) != sizeof(int)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to notify connect thread: %s", strerror(errno)); - } - - return 0; -} - -/* - * connect / outgoing socket management thread - */ - -/* - * _handle_connected_sctp* are called with a global write lock - * from the connect_thread - */ -static void _handle_connected_sctp_socket(knet_handle_t knet_h, int connect_sock) -{ - int err; - unsigned int status, len = sizeof(status); - sctp_connect_link_info_t *info = knet_h->knet_transport_fd_tracker[connect_sock].data; - struct knet_link *kn_link = info->link; - - if (info->close_sock) { - if (_close_connect_socket(knet_h, kn_link) < 0) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to close sock %d from _handle_connected_sctp_socket: %s", connect_sock, strerror(errno)); - return; - } - info->close_sock = 0; - if (_create_connect_socket(knet_h, kn_link) < 0) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to recreate connecting sock! %s", strerror(errno)); - return; - } - } - - _reconnect_socket(knet_h, info->link); - - err = getsockopt(connect_sock, SOL_SOCKET, SO_ERROR, &status, &len); - if (err) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP getsockopt() on connecting socket %d failed: %s", - connect_sock, strerror(errno)); - return; - } - - if (status) { - log_info(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP connect on %d to %s port %s failed: %s", - connect_sock, kn_link->status.dst_ipaddr, kn_link->status.dst_port, - strerror(status)); - - /* - * No need to create a new socket if connect failed, - * just retry connect - */ - return; - } - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP handler fd %d now connected to %s port %s", - connect_sock, - kn_link->status.dst_ipaddr, kn_link->status.dst_port); -} - -static void _handle_connected_sctp_notifications(knet_handle_t knet_h) -{ - int sockfd = -1; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - - if (recv(handle_info->connectsockfd[0], &sockfd, sizeof(int), MSG_DONTWAIT | MSG_NOSIGNAL) != sizeof(int)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Short read on connectsockfd"); - return; - } - - if (_is_valid_fd(knet_h, sockfd) < 1) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received stray notification for connected socket fd error"); - return; - } - - /* - * revalidate sockfd - */ - if ((sockfd < 0) || (sockfd >= KNET_MAX_FDS)) { - return; - } - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Processing connected error on socket: %d", sockfd); - - _handle_connected_sctp_socket(knet_h, sockfd); -} - -static void *_sctp_connect_thread(void *data) -{ - int savederrno; - int i, nev; - knet_handle_t knet_h = (knet_handle_t) data; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - struct epoll_event events[KNET_EPOLL_MAX_EVENTS]; - - set_thread_status(knet_h, KNET_THREAD_SCTP_CONN, KNET_THREAD_STARTED); - - memset(&events, 0, sizeof(events)); - - while (!shutdown_in_progress(knet_h)) { - nev = epoll_wait(handle_info->connect_epollfd, events, KNET_EPOLL_MAX_EVENTS, knet_h->threads_timer_res / 1000); - - /* - * we use timeout to detect if thread is shutting down - */ - if (nev == 0) { - continue; - } - - if (nev < 0) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP connect handler EPOLL ERROR: %s", - strerror(errno)); - continue; - } - - /* - * Sort out which FD has a connection - */ - savederrno = get_global_wrlock(knet_h); - if (savederrno) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to get write lock: %s", - strerror(savederrno)); - continue; - } - - /* - * minor optimization: deduplicate events - * - * in some cases we can receive multiple notifcations - * of the same FD having issues or need handling. - * It's enough to process it once even tho it's safe - * to handle them multiple times. - */ - for (i = 0; i < nev; i++) { - if (events[i].data.fd == handle_info->connectsockfd[0]) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received notification from rx_error for connected socket"); - _handle_connected_sctp_notifications(knet_h); - } else { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received stray notification on connected sockfd %d\n", events[i].data.fd); - } - } - pthread_rwlock_unlock(&knet_h->global_rwlock); - /* - * this thread can generate events for itself. - * we need to sleep in between loops to allow other threads - * to be scheduled - */ - usleep(knet_h->reconnect_int * 1000); - } - - set_thread_status(knet_h, KNET_THREAD_SCTP_CONN, KNET_THREAD_STOPPED); - - return NULL; -} - -/* - * listen/incoming connections management thread - */ - -/* - * Listener received a new connection - * called with a write lock from main thread - */ -static void _handle_incoming_sctp(knet_handle_t knet_h, int listen_sock) -{ - int err = 0, savederrno = 0; - int new_fd; - int i = -1; - sctp_listen_link_info_t *info = knet_h->knet_transport_fd_tracker[listen_sock].data; - struct epoll_event ev; - struct sockaddr_storage ss; - socklen_t sock_len = sizeof(ss); - char addr_str[KNET_MAX_HOST_LEN]; - char port_str[KNET_MAX_PORT_LEN]; - sctp_accepted_link_info_t *accept_info = NULL; - struct knet_host *host; - struct knet_link *kn_link; - int link_idx; - sctp_connect_link_info_t *this_link_connect_info; - sctp_listen_link_info_t *this_link_listen_info; - int pass_acl = 0; - - memset(&ss, 0, sizeof(ss)); - - new_fd = accept(listen_sock, (struct sockaddr *)&ss, &sock_len); - if (new_fd < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Incoming: accept error: %s", strerror(errno)); - goto exit_error; - } - - if (knet_addrtostr(&ss, sizeof(ss), - addr_str, KNET_MAX_HOST_LEN, - port_str, KNET_MAX_PORT_LEN) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Incoming: unable to gather socket info"); - goto exit_error; - } - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Incoming: received connection from: %s port: %s", - addr_str, port_str); - - if (knet_h->use_access_lists) { - for (host = knet_h->host_head; host != NULL; host = host->next) { - for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) { - kn_link = &host->link[link_idx]; - - if ((kn_link->configured) && (kn_link->transport == KNET_TRANSPORT_SCTP)) { - this_link_connect_info = kn_link->transport_link; - this_link_listen_info = this_link_connect_info->listener; - if ((this_link_listen_info->listen_sock == listen_sock) && - (check_validate(knet_h, kn_link, &ss))) { - pass_acl = 1; - break; - } - } - } - if (pass_acl) { - break; - } - } - if (!pass_acl) { - savederrno = EINVAL; - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Connection rejected from %s/%s", addr_str, port_str); - close(new_fd); - errno = savederrno; - return; - } - } - - /* - * Keep a track of all accepted FDs - */ - for (i=0; iaccepted_socks[i] == -1) { - info->accepted_socks[i] = new_fd; - break; - } - } - - if (i == MAX_ACCEPTED_SOCKS) { - errno = EBUSY; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Incoming: too many connections!"); - goto exit_error; - } - - if (_configure_common_socket(knet_h, new_fd, 0, "SCTP incoming") < 0) { /* Inherit flags from listener? */ - savederrno = errno; - err = -1; - goto exit_error; - } - - if (_enable_sctp_notifications(knet_h, new_fd, "Incoming connection") < 0) { - savederrno = errno; - err = -1; - goto exit_error; - } - - accept_info = malloc(sizeof(sctp_accepted_link_info_t)); - if (!accept_info) { - savederrno = errno; - err = -1; - goto exit_error; - } - memset(accept_info, 0, sizeof(sctp_accepted_link_info_t)); - - accept_info->link_info = info; - - if (_set_fd_tracker(knet_h, new_fd, KNET_TRANSPORT_SCTP, SCTP_ACCEPTED_LINK_INFO, - knet_h->knet_transport_fd_tracker[listen_sock].sockaddr_len, - accept_info, -1) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set fd tracker: %s", - strerror(errno)); - goto exit_error; - } - - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = new_fd; - if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_ADD, new_fd, &ev)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Incoming: unable to add accepted socket %d to epoll pool: %s", - new_fd, strerror(errno)); - goto exit_error; - } - info->on_rx_epoll = 1; - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Incoming: accepted new fd %d for %s/%s (listen fd: %d). index: %d", - new_fd, addr_str, port_str, info->listen_sock, i); - -exit_error: - if (err) { - if ((i >= 0) && (i < MAX_ACCEPTED_SOCKS)) { - info->accepted_socks[i] = -1; - } - /* - * check the error to make coverity scan happy. - * _set_fd_tracker cannot fail at this stage - */ - if (_set_fd_tracker(knet_h, new_fd, KNET_MAX_TRANSPORTS, SCTP_NO_LINK_INFO, 0, NULL, -1) < 0){ - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to update fdtracker for socket %d", new_fd); - } - free(accept_info); - if (new_fd >= 0) { - close(new_fd); - } - } - errno = savederrno; - return; -} - -/* - * Listen thread received a notification of a bad socket that needs closing - * called with a write lock from main thread - */ -static void _handle_listen_sctp_errors(knet_handle_t knet_h) -{ - int sockfd = -1; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - sctp_accepted_link_info_t *accept_info; - sctp_listen_link_info_t *info; - struct knet_host *host; - int link_idx; - int i; - - if (recv(handle_info->listensockfd[0], &sockfd, sizeof(int), MSG_DONTWAIT | MSG_NOSIGNAL) != sizeof(int)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Short read on listensockfd"); - return; - } - - if (_is_valid_fd(knet_h, sockfd) < 1) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received stray notification for listen socket fd error"); - return; - } - - /* - * revalidate sockfd - */ - if ((sockfd < 0) || (sockfd >= KNET_MAX_FDS)) { - return; - } - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Processing listen error on socket: %d", sockfd); - - accept_info = knet_h->knet_transport_fd_tracker[sockfd].data; - info = accept_info->link_info; - - /* - * clear all links using this accepted socket as - * outbound dynamically connected socket - */ - - for (host = knet_h->host_head; host != NULL; host = host->next) { - for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) { - if ((host->link[link_idx].dynamic == KNET_LINK_DYNIP) && - (host->link[link_idx].outsock == sockfd)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Found dynamic connection on host %d link %d (%d)", - host->host_id, link_idx, sockfd); - host->link[link_idx].status.dynconnected = 0; - host->link[link_idx].transport_connected = 0; - host->link[link_idx].outsock = 0; - memset(&host->link[link_idx].dst_addr, 0, sizeof(struct sockaddr_storage)); - } - } - } - - for (i=0; iaccepted_socks[i]) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Closing accepted socket %d", sockfd); - /* - * check the error to make coverity scan happy. - * _set_fd_tracker cannot fail at this stage - */ - if (_set_fd_tracker(knet_h, sockfd, KNET_MAX_TRANSPORTS, SCTP_NO_LINK_INFO, 0, NULL, -1) < 0) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to update fdtracker for socket %d", sockfd); - } - info->accepted_socks[i] = -1; - free(accept_info); - close(sockfd); - break; /* Keeps covscan happy */ - } - } -} - -static void *_sctp_listen_thread(void *data) -{ - int savederrno; - int i, nev; - knet_handle_t knet_h = (knet_handle_t) data; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - struct epoll_event events[KNET_EPOLL_MAX_EVENTS]; - - set_thread_status(knet_h, KNET_THREAD_SCTP_LISTEN, KNET_THREAD_STARTED); - - memset(&events, 0, sizeof(events)); - - while (!shutdown_in_progress(knet_h)) { - nev = epoll_wait(handle_info->listen_epollfd, events, KNET_EPOLL_MAX_EVENTS, knet_h->threads_timer_res / 1000); - - /* - * we use timeout to detect if thread is shutting down - */ - if (nev == 0) { - continue; - } - - if (nev < 0) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP listen handler EPOLL ERROR: %s", - strerror(errno)); - continue; - } - - savederrno = get_global_wrlock(knet_h); - if (savederrno) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to get write lock: %s", - strerror(savederrno)); - continue; - } - /* - * Sort out which FD has an incoming connection - */ - for (i = 0; i < nev; i++) { - if (events[i].data.fd == handle_info->listensockfd[0]) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received notification from rx_error for listener/accepted socket"); - _handle_listen_sctp_errors(knet_h); - } else { - if (_is_valid_fd(knet_h, events[i].data.fd) == 1) { - _handle_incoming_sctp(knet_h, events[i].data.fd); - } else { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Received listen notification from invalid socket"); - } - } - - } - pthread_rwlock_unlock(&knet_h->global_rwlock); - } - - set_thread_status(knet_h, KNET_THREAD_SCTP_LISTEN, KNET_THREAD_STOPPED); - - return NULL; -} - -/* - * sctp_link_listener_start/stop are called in global write lock - * context from set_config and clear_config. - */ -static sctp_listen_link_info_t *sctp_link_listener_start(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int err = 0, savederrno = 0; - int listen_sock = -1; - struct epoll_event ev; - sctp_listen_link_info_t *info = NULL; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - - /* - * Only allocate a new listener if src address is different - */ - qb_list_for_each_entry(info, &handle_info->listen_links_list, list) { - if (memcmp(&info->src_address, &kn_link->src_addr, sizeof(struct sockaddr_storage)) == 0) { - if ((check_add(knet_h, kn_link, -1, - &kn_link->dst_addr, &kn_link->dst_addr, CHECK_TYPE_ADDRESS, CHECK_ACCEPT) < 0) && (errno != EEXIST)) { - return NULL; - } - return info; - } - } - - info = malloc(sizeof(sctp_listen_link_info_t)); - if (!info) { - err = -1; - goto exit_error; - } - - memset(info, 0, sizeof(sctp_listen_link_info_t)); - - memset(info->accepted_socks, -1, sizeof(info->accepted_socks)); - memmove(&info->src_address, &kn_link->src_addr, sizeof(struct sockaddr_storage)); - - listen_sock = socket(kn_link->src_addr.ss_family, SOCK_STREAM, IPPROTO_SCTP); - if (listen_sock < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to create listener socket: %s", - strerror(savederrno)); - goto exit_error; - } - - if (_configure_sctp_socket(knet_h, listen_sock, &kn_link->src_addr, kn_link->flags, "SCTP listener") < 0) { - savederrno = errno; - err = -1; - goto exit_error; - } - - if (bind(listen_sock, (struct sockaddr *)&kn_link->src_addr, sockaddr_len(&kn_link->src_addr)) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to bind listener socket: %s", - strerror(savederrno)); - goto exit_error; - } - - if (listen(listen_sock, 5) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to listen on listener socket: %s", - strerror(savederrno)); - goto exit_error; - } - - if (_set_fd_tracker(knet_h, listen_sock, KNET_TRANSPORT_SCTP, SCTP_LISTENER_LINK_INFO, sockaddr_len(&kn_link->src_addr), info, -1) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set fd tracker: %s", - strerror(savederrno)); - goto exit_error; - } - - if ((check_add(knet_h, kn_link, -1, - &kn_link->dst_addr, &kn_link->dst_addr, CHECK_TYPE_ADDRESS, CHECK_ACCEPT) < 0) && (errno != EEXIST)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to configure default access lists: %s", - strerror(savederrno)); - goto exit_error; - } - - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = listen_sock; - if (epoll_ctl(handle_info->listen_epollfd, EPOLL_CTL_ADD, listen_sock, &ev)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to add listener to epoll pool: %s", - strerror(savederrno)); - goto exit_error; - } - info->on_listener_epoll = 1; - - info->listen_sock = listen_sock; - qb_list_add(&info->list, &handle_info->listen_links_list); - - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Listening on fd %d for %s:%s", listen_sock, kn_link->status.src_ipaddr, kn_link->status.src_port); - -exit_error: - if (err) { - if ((info) && (info->on_listener_epoll)) { - epoll_ctl(handle_info->listen_epollfd, EPOLL_CTL_DEL, listen_sock, &ev); - } - if (listen_sock >= 0) { - check_rmall(knet_h, kn_link); - close(listen_sock); - } - if (info) { - free(info); - info = NULL; - } - } - errno = savederrno; - return info; -} - -static int sctp_link_listener_stop(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int err = 0, savederrno = 0; - int found = 0, i; - struct knet_host *host; - int link_idx; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - sctp_connect_link_info_t *this_link_info = kn_link->transport_link; - sctp_listen_link_info_t *info = this_link_info->listener; - sctp_connect_link_info_t *link_info; - struct epoll_event ev; - - for (host = knet_h->host_head; host != NULL; host = host->next) { - for (link_idx = 0; link_idx < KNET_MAX_LINK; link_idx++) { - if (&host->link[link_idx] == kn_link) - continue; - - link_info = host->link[link_idx].transport_link; - if ((link_info) && - (link_info->listener == info)) { - found = 1; - break; - } - } - } - - if ((check_rm(knet_h, kn_link, - &kn_link->dst_addr, &kn_link->dst_addr, CHECK_TYPE_ADDRESS, CHECK_ACCEPT) < 0) && (errno != ENOENT)) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to remove default access lists for %d", info->listen_sock); - } - - if (found) { - this_link_info->listener = NULL; - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP listener socket %d still in use", info->listen_sock); - savederrno = EBUSY; - err = -1; - goto exit_error; - } - - if (info->on_listener_epoll) { - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = info->listen_sock; - if (epoll_ctl(handle_info->listen_epollfd, EPOLL_CTL_DEL, info->listen_sock, &ev)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to remove listener to epoll pool: %s", - strerror(savederrno)); - goto exit_error; - } - info->on_listener_epoll = 0; - } - - if (_set_fd_tracker(knet_h, info->listen_sock, KNET_MAX_TRANSPORTS, SCTP_NO_LINK_INFO, 0, NULL, -1) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set fd tracker: %s", - strerror(savederrno)); - goto exit_error; - } - - check_rmall(knet_h, kn_link); - close(info->listen_sock); - - for (i=0; i< MAX_ACCEPTED_SOCKS; i++) { - if (info->accepted_socks[i] > -1) { - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = info->accepted_socks[i]; - if (epoll_ctl(knet_h->recv_from_links_epollfd, EPOLL_CTL_DEL, info->accepted_socks[i], &ev)) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to remove EOFed socket from epoll pool: %s", - strerror(errno)); - } - info->on_rx_epoll = 0; - free(knet_h->knet_transport_fd_tracker[info->accepted_socks[i]].data); - close(info->accepted_socks[i]); - if (_set_fd_tracker(knet_h, info->accepted_socks[i], KNET_MAX_TRANSPORTS, SCTP_NO_LINK_INFO, 0, NULL, -1) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set fd tracker: %s", - strerror(savederrno)); - goto exit_error; - } - info->accepted_socks[i] = -1; - } - } - - qb_list_del(&info->list); - free(info); - this_link_info->listener = NULL; - -exit_error: - errno = savederrno; - return err; -} - -/* - * Links config/clear. Both called with global wrlock from link_set_config/clear_config - */ -int sctp_transport_link_set_config(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int savederrno = 0, err = 0; - sctp_connect_link_info_t *info; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - - info = malloc(sizeof(sctp_connect_link_info_t)); - if (!info) { - goto exit_error; - } - - memset(info, 0, sizeof(sctp_connect_link_info_t)); - - kn_link->transport_link = info; - info->link = kn_link; - - memmove(&info->dst_address, &kn_link->dst_addr, sizeof(struct sockaddr_storage)); - info->connect_sock = -1; - - info->listener = sctp_link_listener_start(knet_h, kn_link); - if (!info->listener) { - savederrno = errno; - err = -1; - goto exit_error; - } - - if (kn_link->dynamic == KNET_LINK_STATIC) { - if (_create_connect_socket(knet_h, kn_link) < 0) { - savederrno = errno; - err = -1; - goto exit_error; - } - kn_link->outsock = info->connect_sock; - } - - qb_list_add(&info->list, &handle_info->connect_links_list); - -exit_error: - if (err) { - if (info) { - if (info->connect_sock >= 0) { - close(info->connect_sock); - } - if (info->listener) { - sctp_link_listener_stop(knet_h, kn_link); - } - kn_link->transport_link = NULL; - free(info); - } - } - errno = savederrno; - return err; -} - -/* - * called with global wrlock - */ -int sctp_transport_link_clear_config(knet_handle_t knet_h, struct knet_link *kn_link) -{ - int err = 0, savederrno = 0; - sctp_connect_link_info_t *info; - - if (!kn_link) { - errno = EINVAL; - return -1; - } - - info = kn_link->transport_link; - - if (!info) { - errno = EINVAL; - return -1; - } - - if ((sctp_link_listener_stop(knet_h, kn_link) <0) && (errno != EBUSY)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to remove listener transport: %s", - strerror(savederrno)); - goto exit_error; - } - - if (_close_connect_socket(knet_h, kn_link) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to close connected socket: %s", - strerror(savederrno)); - goto exit_error; - } - - qb_list_del(&info->list); - - free(info); - kn_link->transport_link = NULL; - -exit_error: - errno = savederrno; - return err; -} - -/* - * transport_free and transport_init are - * called only from knet_handle_new and knet_handle_free. - * all resources (hosts/links) should have been already freed at this point - * and they are called in a write locked context, hence they - * don't need their own locking. - */ - -int sctp_transport_free(knet_handle_t knet_h) -{ - sctp_handle_info_t *handle_info; - void *thread_status; - struct epoll_event ev; - - if (!knet_h->transports[KNET_TRANSPORT_SCTP]) { - errno = EINVAL; - return -1; - } - - handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - - /* - * keep it here while we debug list usage and such - */ - if (!qb_list_empty(&handle_info->listen_links_list)) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Internal error. listen links list is not empty"); - } - if (!qb_list_empty(&handle_info->connect_links_list)) { - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Internal error. connect links list is not empty"); - } - - if (handle_info->listen_thread) { - pthread_cancel(handle_info->listen_thread); - pthread_join(handle_info->listen_thread, &thread_status); - } - - if (handle_info->connect_thread) { - pthread_cancel(handle_info->connect_thread); - pthread_join(handle_info->connect_thread, &thread_status); - } - - if (handle_info->listensockfd[0] >= 0) { - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = handle_info->listensockfd[0]; - epoll_ctl(handle_info->listen_epollfd, EPOLL_CTL_DEL, handle_info->listensockfd[0], &ev); - } - - if (handle_info->connectsockfd[0] >= 0) { - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = handle_info->connectsockfd[0]; - epoll_ctl(handle_info->connect_epollfd, EPOLL_CTL_DEL, handle_info->connectsockfd[0], &ev); - } - - _close_socketpair(knet_h, handle_info->connectsockfd); - _close_socketpair(knet_h, handle_info->listensockfd); - - if (handle_info->listen_epollfd >= 0) { - close(handle_info->listen_epollfd); - } - - if (handle_info->connect_epollfd >= 0) { - close(handle_info->connect_epollfd); - } - - free(handle_info->event_subscribe_buffer); - free(handle_info); - knet_h->transports[KNET_TRANSPORT_SCTP] = NULL; - - return 0; -} - -static int _sctp_subscribe_init(knet_handle_t knet_h) -{ - int test_socket, savederrno; - sctp_handle_info_t *handle_info = knet_h->transports[KNET_TRANSPORT_SCTP]; - char dummy_events[100]; - struct sctp_event_subscribe *events; - /* Below we set the first 6 fields of this expanding struct. - * SCTP_EVENTS is deprecated, but SCTP_EVENT is not available - * on Linux; on the other hand, FreeBSD and old Linux does not - * accept small transfers, so we can't simply use this minimum - * everywhere. Thus we query and store the native size. */ - const unsigned int subscribe_min = 6; - - test_socket = socket(PF_INET, SOCK_STREAM, IPPROTO_SCTP); - if (test_socket < 0) { - if (errno == EPROTONOSUPPORT) { - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "SCTP not supported, skipping initialization"); - return 0; - } - savederrno = errno; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to create test socket: %s", - strerror(savederrno)); - return savederrno; - } - handle_info->event_subscribe_kernel_size = sizeof dummy_events; - if (getsockopt(test_socket, IPPROTO_SCTP, SCTP_EVENTS, &dummy_events, - &handle_info->event_subscribe_kernel_size)) { - close(test_socket); - savederrno = errno; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to query kernel size of struct sctp_event_subscribe: %s", - strerror(savederrno)); - return savederrno; - } - close(test_socket); - if (handle_info->event_subscribe_kernel_size < subscribe_min) { - savederrno = ERANGE; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, - "No kernel support for the necessary notifications: struct sctp_event_subscribe is %u bytes, %u needed", - handle_info->event_subscribe_kernel_size, subscribe_min); - return savederrno; - } - events = malloc(handle_info->event_subscribe_kernel_size); - if (!events) { - savederrno = errno; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, - "Failed to allocate event subscribe buffer: %s", strerror(savederrno)); - return savederrno; - } - memset(events, 0, handle_info->event_subscribe_kernel_size); - events->sctp_data_io_event = 1; - events->sctp_association_event = 1; - events->sctp_address_event = 1; - events->sctp_send_failure_event = 1; - events->sctp_peer_error_event = 1; - events->sctp_shutdown_event = 1; - handle_info->event_subscribe_buffer = (char *)events; - log_debug(knet_h, KNET_SUB_TRANSP_SCTP, "Size of struct sctp_event_subscribe is %u in kernel, %zu in user space", - handle_info->event_subscribe_kernel_size, sizeof(struct sctp_event_subscribe)); - return 0; -} - -int sctp_transport_init(knet_handle_t knet_h) -{ - int err = 0, savederrno = 0; - sctp_handle_info_t *handle_info; - struct epoll_event ev; - - if (knet_h->transports[KNET_TRANSPORT_SCTP]) { - errno = EEXIST; - return -1; - } - - handle_info = malloc(sizeof(sctp_handle_info_t)); - if (!handle_info) { - return -1; - } - - memset(handle_info, 0,sizeof(sctp_handle_info_t)); - - knet_h->transports[KNET_TRANSPORT_SCTP] = handle_info; - - savederrno = _sctp_subscribe_init(knet_h); - if (savederrno) { - err = -1; - goto exit_fail; - } - - qb_list_init(&handle_info->listen_links_list); - qb_list_init(&handle_info->connect_links_list); - - handle_info->listen_epollfd = epoll_create(KNET_EPOLL_MAX_EVENTS + 1); - if (handle_info->listen_epollfd < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to create epoll listen fd: %s", - strerror(savederrno)); - goto exit_fail; - } - - if (_fdset_cloexec(handle_info->listen_epollfd)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set CLOEXEC on listen_epollfd: %s", - strerror(savederrno)); - goto exit_fail; - } - - handle_info->connect_epollfd = epoll_create(KNET_EPOLL_MAX_EVENTS + 1); - if (handle_info->connect_epollfd < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to create epoll connect fd: %s", - strerror(savederrno)); - goto exit_fail; - } - - if (_fdset_cloexec(handle_info->connect_epollfd)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to set CLOEXEC on connect_epollfd: %s", - strerror(savederrno)); - goto exit_fail; - } - - if (_init_socketpair(knet_h, handle_info->connectsockfd) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to init connect socketpair: %s", - strerror(savederrno)); - goto exit_fail; - } - - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = handle_info->connectsockfd[0]; - if (epoll_ctl(handle_info->connect_epollfd, EPOLL_CTL_ADD, handle_info->connectsockfd[0], &ev)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to add connectsockfd[0] to connect epoll pool: %s", - strerror(savederrno)); - goto exit_fail; - } - - if (_init_socketpair(knet_h, handle_info->listensockfd) < 0) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to init listen socketpair: %s", - strerror(savederrno)); - goto exit_fail; - } - - memset(&ev, 0, sizeof(struct epoll_event)); - ev.events = EPOLLIN; - ev.data.fd = handle_info->listensockfd[0]; - if (epoll_ctl(handle_info->listen_epollfd, EPOLL_CTL_ADD, handle_info->listensockfd[0], &ev)) { - savederrno = errno; - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to add listensockfd[0] to listen epoll pool: %s", - strerror(savederrno)); - goto exit_fail; - } - - /* - * Start connect & listener threads - */ - set_thread_status(knet_h, KNET_THREAD_SCTP_LISTEN, KNET_THREAD_REGISTERED); - savederrno = pthread_create(&handle_info->listen_thread, 0, _sctp_listen_thread, (void *) knet_h); - if (savederrno) { - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to start sctp listen thread: %s", - strerror(savederrno)); - goto exit_fail; - } - - set_thread_status(knet_h, KNET_THREAD_SCTP_CONN, KNET_THREAD_REGISTERED); - savederrno = pthread_create(&handle_info->connect_thread, 0, _sctp_connect_thread, (void *) knet_h); - if (savederrno) { - err = -1; - log_err(knet_h, KNET_SUB_TRANSP_SCTP, "Unable to start sctp connect thread: %s", - strerror(savederrno)); - goto exit_fail; - } - -exit_fail: - if (err < 0) { - sctp_transport_free(knet_h); - } - errno = savederrno; - return err; -} - -int sctp_transport_link_dyn_connect(knet_handle_t knet_h, int sockfd, struct knet_link *kn_link) -{ - kn_link->outsock = sockfd; - kn_link->status.dynconnected = 1; - kn_link->transport_connected = 1; - return 0; -} -#endif diff --git a/libknet/transport_sctp.h b/libknet/transport_sctp.h deleted file mode 100644 index dfa5d58f..00000000 --- a/libknet/transport_sctp.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2017-2025 Red Hat, Inc. All rights reserved. - * - * Authors: Fabio M. Di Nitto - * - * This software licensed under LGPL-2.0+ - */ - -#include "config.h" - -#include "internals.h" - -#ifndef __KNET_TRANSPORT_SCTP_H__ -#define __KNET_TRANSPORT_SCTP_H__ - -/* - * https://en.wikipedia.org/wiki/SCTP_packet_structure - */ - -#define KNET_PMTUD_SCTP_OVERHEAD_COMMON 12 -#define KNET_PMTUD_SCTP_OVERHEAD_DATA_CHUNK 16 -#define KNET_PMTUD_SCTP_OVERHEAD KNET_PMTUD_SCTP_OVERHEAD_COMMON + KNET_PMTUD_SCTP_OVERHEAD_DATA_CHUNK - -#ifdef HAVE_NETINET_SCTP_H - -int sctp_transport_link_set_config(knet_handle_t knet_h, struct knet_link *kn_link); -int sctp_transport_link_clear_config(knet_handle_t knet_h, struct knet_link *kn_link); -int sctp_transport_free(knet_handle_t knet_h); -int sctp_transport_init(knet_handle_t knet_h); -transport_sock_error_t sctp_transport_rx_sock_error(knet_handle_t knet_h, int sockfd, int recv_err, int recv_errno); -transport_sock_error_t sctp_transport_tx_sock_error(knet_handle_t knet_h, int sockfd, int subsys, int recv_err, int recv_errno); -transport_rx_isdata_t sctp_transport_rx_is_data(knet_handle_t knet_h, int sockfd, struct knet_mmsghdr *msg); -int sctp_transport_link_dyn_connect(knet_handle_t knet_h, int sockfd, struct knet_link *kn_link); -int sctp_transport_link_is_down(knet_handle_t knet_h, struct knet_link *kn_link); - -#endif - -#endif diff --git a/libknet/transports.c b/libknet/transports.c index fd3e6bb3..5dce2a7a 100644 --- a/libknet/transports.c +++ b/libknet/transports.c @@ -1,290 +1,283 @@ /* * Copyright (C) 2017-2025 Red Hat, Inc. All rights reserved. * * Author: Fabio M. Di Nitto * * This software licensed under LGPL-2.0+ */ #include "config.h" #include #include #include #include #include #include #include "libknet.h" #include "compat.h" #include "host.h" #include "link.h" #include "logging.h" #include "common.h" #include "transports.h" #include "transport_loopback.h" #include "transport_udp.h" -#include "transport_sctp.h" #include "threads_common.h" #define empty_module 0, -1, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }, static knet_transport_ops_t transport_modules_cmd[KNET_MAX_TRANSPORTS] = { { "LOOPBACK", KNET_TRANSPORT_LOOPBACK, 1, TRANSPORT_PROTO_LOOPBACK, USE_NO_ACL, TRANSPORT_PROTO_NOT_CONNECTION_ORIENTED, KNET_PMTUD_LOOPBACK_OVERHEAD, loopback_transport_init, loopback_transport_free, loopback_transport_link_set_config, loopback_transport_link_clear_config, loopback_transport_link_dyn_connect, loopback_transport_rx_sock_error, loopback_transport_tx_sock_error, loopback_transport_rx_is_data, loopback_transport_link_is_down }, { "UDP", KNET_TRANSPORT_UDP, 1, TRANSPORT_PROTO_IP_PROTO, USE_GENERIC_ACL, TRANSPORT_PROTO_NOT_CONNECTION_ORIENTED, KNET_PMTUD_UDP_OVERHEAD, udp_transport_init, udp_transport_free, udp_transport_link_set_config, udp_transport_link_clear_config, udp_transport_link_dyn_connect, udp_transport_rx_sock_error, udp_transport_tx_sock_error, udp_transport_rx_is_data, udp_transport_link_is_down }, - { "SCTP", KNET_TRANSPORT_SCTP, -#ifdef HAVE_NETINET_SCTP_H - 1, TRANSPORT_PROTO_IP_PROTO, USE_PROTO_ACL, TRANSPORT_PROTO_IS_CONNECTION_ORIENTED, KNET_PMTUD_SCTP_OVERHEAD, sctp_transport_init, sctp_transport_free, sctp_transport_link_set_config, sctp_transport_link_clear_config, sctp_transport_link_dyn_connect, sctp_transport_rx_sock_error, sctp_transport_tx_sock_error, sctp_transport_rx_is_data, sctp_transport_link_is_down }, -#else -empty_module -#endif { NULL, KNET_MAX_TRANSPORTS, empty_module }; /* * transport wrappers */ int start_all_transports(knet_handle_t knet_h) { int idx = 0, savederrno = 0, err = 0; while (transport_modules_cmd[idx].transport_name != NULL) { if (transport_modules_cmd[idx].built_in) { if (transport_modules_cmd[idx].transport_init(knet_h) < 0) { savederrno = errno; log_err(knet_h, KNET_SUB_HANDLE, "Failed to allocate transport handle for %s: %s", transport_modules_cmd[idx].transport_name, strerror(savederrno)); err = -1; goto out; } } idx++; } out: errno = savederrno; return err; } void stop_all_transports(knet_handle_t knet_h) { int idx = 0; while (transport_modules_cmd[idx].transport_name != NULL) { if (transport_modules_cmd[idx].built_in) { transport_modules_cmd[idx].transport_free(knet_h); } idx++; } } int transport_link_set_config(knet_handle_t knet_h, struct knet_link *kn_link, uint8_t transport) { if (!transport_modules_cmd[transport].built_in) { errno = EINVAL; return -1; } kn_link->transport_connected = 0; kn_link->transport = transport; kn_link->proto_overhead = transport_modules_cmd[transport].transport_mtu_overhead; return transport_modules_cmd[transport].transport_link_set_config(knet_h, kn_link); } int transport_link_clear_config(knet_handle_t knet_h, struct knet_link *kn_link) { return transport_modules_cmd[kn_link->transport].transport_link_clear_config(knet_h, kn_link); } int transport_link_dyn_connect(knet_handle_t knet_h, int sockfd, struct knet_link *kn_link) { return transport_modules_cmd[kn_link->transport].transport_link_dyn_connect(knet_h, sockfd, kn_link); } int transport_rx_sock_error(knet_handle_t knet_h, uint8_t transport, int sockfd, int recv_err, int recv_errno) { return transport_modules_cmd[transport].transport_rx_sock_error(knet_h, sockfd, recv_err, recv_errno); } int transport_tx_sock_error(knet_handle_t knet_h, uint8_t transport, int sockfd, int subsys, int recv_err, int recv_errno) { return transport_modules_cmd[transport].transport_tx_sock_error(knet_h, sockfd, subsys, recv_err, recv_errno); } int transport_rx_is_data(knet_handle_t knet_h, uint8_t transport, int sockfd, struct knet_mmsghdr *msg) { return transport_modules_cmd[transport].transport_rx_is_data(knet_h, sockfd, msg); } int transport_get_proto(knet_handle_t knet_h, uint8_t transport) { return transport_modules_cmd[transport].transport_protocol; } int transport_get_acl_type(knet_handle_t knet_h, uint8_t transport) { return transport_modules_cmd[transport].transport_acl_type; } int transport_get_connection_oriented(knet_handle_t knet_h, uint8_t transport) { return transport_modules_cmd[transport].transport_is_connection_oriented; } int transport_link_is_down(knet_handle_t knet_h, struct knet_link *kn_link) { return transport_modules_cmd[kn_link->transport].transport_link_is_down(knet_h, kn_link); } /* * public api */ int knet_get_transport_list(struct knet_transport_info *transport_list, size_t *transport_list_entries) { int err = 0; int idx = 0; int outidx = 0; if (!transport_list_entries) { errno = EINVAL; return -1; } while (transport_modules_cmd[idx].transport_name != NULL) { if (transport_modules_cmd[idx].built_in) { if (transport_list) { transport_list[outidx].name = transport_modules_cmd[idx].transport_name; transport_list[outidx].id = transport_modules_cmd[idx].transport_id; } outidx++; } idx++; } *transport_list_entries = outidx; if (!err) errno = 0; return err; } const char *knet_get_transport_name_by_id(uint8_t transport) { int savederrno = 0; const char *name = NULL; if (transport == KNET_MAX_TRANSPORTS) { errno = EINVAL; return name; } if ((transport_modules_cmd[transport].transport_name) && (transport_modules_cmd[transport].built_in)) { name = transport_modules_cmd[transport].transport_name; } else { savederrno = ENOENT; } errno = name ? 0 : savederrno; return name; } uint8_t knet_get_transport_id_by_name(const char *name) { int savederrno = 0; uint8_t err = KNET_MAX_TRANSPORTS; int i, found; if (!name) { errno = EINVAL; return err; } i = 0; found = 0; while (transport_modules_cmd[i].transport_name != NULL) { if (transport_modules_cmd[i].built_in) { if (!strcmp(transport_modules_cmd[i].transport_name, name)) { err = transport_modules_cmd[i].transport_id; found = 1; break; } } i++; } if (!found) { savederrno = EINVAL; } errno = err == KNET_MAX_TRANSPORTS ? savederrno : 0; return err; } int knet_handle_set_transport_reconnect_interval(knet_handle_t knet_h, uint32_t msecs) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (!msecs) { errno = EINVAL; return -1; } if (msecs < 1000) { log_warn(knet_h, KNET_SUB_HANDLE, "reconnect internval below 1 sec (%u msecs) might be too aggressive", msecs); } if (msecs > 60000) { log_warn(knet_h, KNET_SUB_HANDLE, "reconnect internval above 1 minute (%u msecs) could cause long delays in network convergiance", msecs); } savederrno = get_global_wrlock(knet_h); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } knet_h->reconnect_int = msecs; pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; } int knet_handle_get_transport_reconnect_interval(knet_handle_t knet_h, uint32_t *msecs) { int savederrno = 0; if (!_is_valid_handle(knet_h)) { return -1; } if (!msecs) { errno = EINVAL; return -1; } savederrno = pthread_rwlock_rdlock(&knet_h->global_rwlock); if (savederrno) { log_err(knet_h, KNET_SUB_HANDLE, "Unable to get read lock: %s", strerror(savederrno)); errno = savederrno; return -1; } *msecs = knet_h->reconnect_int; pthread_rwlock_unlock(&knet_h->global_rwlock); errno = 0; return 0; }