From be2678cddce7f12d4a984423291d9acc353508c0 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 1/9] BUILD: Add macro for checking python3 modules --- diff --git a/src/external/python.m4 b/src/external/python.m4 index b45f933..8bb352a 100644 --- a/src/external/python.m4 +++ b/src/external/python.m4 @@ -126,3 +126,30 @@ AC_DEFUN([AM_PYTHON2_MODULE],[ fi fi ]) + +AC_DEFUN([AM_PYTHON3_MODULE],[ + if test x"$PYTHON3" = x; then + if test -n "$2"; then + AC_MSG_ERROR([cannot look for $1 module: Python 3 not found]) + else + AC_MSG_NOTICE([cannot look for $1 module: Python 3 not found]) + eval AS_TR_CPP(HAVE_PY3MOD_$1)=no + fi + else + AC_MSG_CHECKING($(basename $PYTHON3) module: $1) + $PYTHON3 -c "import $1" 2>/dev/null + if test $? -eq 0; then + AC_MSG_RESULT(yes) + eval AS_TR_CPP(HAVE_PY3MOD_$1)=yes + else + AC_MSG_RESULT(no) + eval AS_TR_CPP(HAVE_PY3MOD_$1)=no + # + if test -n "$2" + then + AC_MSG_ERROR(failed to find required module $1) + exit 1 + fi + fi + fi +]) From bf892aad4a5d898cc6d4fd3dd00f4c9349cd963c Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 2/9] BUILD: Fix typo of detecting python module for intgcheck --- diff --git a/src/external/intgcheck.m4 b/src/external/intgcheck.m4 index 05b3616..09f267f 100644 --- a/src/external/intgcheck.m4 +++ b/src/external/intgcheck.m4 @@ -32,7 +32,7 @@ AC_DEFUN([SSS_ENABLE_INTGCHECK_REQS], [ SSS_INTGCHECK_REQ([HAVE_PYTHON2], [python2]) SSS_INTGCHECK_REQ([HAVE_PYTEST2], [pytest2]) SSS_INTGCHECK_REQ([HAVE_PY2MOD_LDAP], [python-ldap]) - SSS_INTGCHECK_REQ([HAVE_PY2MOD_LDAP], [pyldb]) + SSS_INTGCHECK_REQ([HAVE_PY2MOD_LDB], [pyldb]) fi ]) From 0d166c6bd1d5237c66fd9ea9cf28a24d218bdea0 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 3/9] BUILD: Move checking of python2 modules for intgcheck --- diff --git a/configure.ac b/configure.ac index e330652..64ca2c0 100644 --- a/configure.ac +++ b/configure.ac @@ -385,8 +385,6 @@ AM_CONDITIONAL([BUILD_PYTHON_BINDINGS], [test x"$with_python2_bindings" = xyes \ -o x"$with_python3_bindings" = xyes]) -AM_PYTHON2_MODULE([ldap]) - if test x$HAVE_SELINUX != x; then AM_CHECK_SELINUX fi diff --git a/src/external/intgcheck.m4 b/src/external/intgcheck.m4 index 09f267f..52c2e1c 100644 --- a/src/external/intgcheck.m4 +++ b/src/external/intgcheck.m4 @@ -29,6 +29,13 @@ AC_DEFUN([SSS_ENABLE_INTGCHECK_REQS], [ SSS_INTGCHECK_REQ([HAVE_SLAPD], [slapd]) SSS_INTGCHECK_REQ([HAVE_LDAPMODIFY], [ldapmodify]) SSS_INTGCHECK_REQ([HAVE_FAKEROOT], [fakeroot]) + + AM_PYTHON2_MODULE([ldap]) + AM_PYTHON2_MODULE([ldb]) + AM_PYTHON2_MODULE([requests]) + AM_PYTHON2_MODULE([dbus]) + AM_PYTHON2_MODULE([psutil]) + SSS_INTGCHECK_REQ([HAVE_PYTHON2], [python2]) SSS_INTGCHECK_REQ([HAVE_PYTEST2], [pytest2]) SSS_INTGCHECK_REQ([HAVE_PY2MOD_LDAP], [python-ldap]) From fd5b2709cf94eaa34b6b0f0a8012035bf9dde0bf Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 4/9] BUILD: Add macro for checking pytest for intgcheck --- diff --git a/src/external/intgcheck.m4 b/src/external/intgcheck.m4 index 52c2e1c..325e8ea 100644 --- a/src/external/intgcheck.m4 +++ b/src/external/intgcheck.m4 @@ -30,6 +30,7 @@ AC_DEFUN([SSS_ENABLE_INTGCHECK_REQS], [ SSS_INTGCHECK_REQ([HAVE_LDAPMODIFY], [ldapmodify]) SSS_INTGCHECK_REQ([HAVE_FAKEROOT], [fakeroot]) + SSS_CHECK_PYTEST([$PYTHON2], [PY2_PYTEST]) AM_PYTHON2_MODULE([ldap]) AM_PYTHON2_MODULE([ldb]) AM_PYTHON2_MODULE([requests]) diff --git a/src/external/python.m4 b/src/external/python.m4 index 8bb352a..c200544 100644 --- a/src/external/python.m4 +++ b/src/external/python.m4 @@ -153,3 +153,31 @@ AC_DEFUN([AM_PYTHON3_MODULE],[ fi fi ]) + +dnl SYNOPSIS +dnl +dnl SSS_CHECK_PYTEST(python_interpreter, have_suffix) +dnl +dnl DESCRIPTION +dnl +dnl Checks for pytest +AC_DEFUN([SSS_CHECK_PYTEST],[ + if test x"$1" = x; then + if test -n "$2"; then + AC_MSG_ERROR([cannot look for pytest: $(basename $1) not found]) + else + AC_MSG_NOTICE([cannot look for pytest module: $(basename $1) not found]) + eval AS_TR_CPP(HAVE_$2)=no + fi + else + AC_MSG_CHECKING($(basename $1) pytest) + $1 -m pytest --version 2>/dev/null + if test $? -eq 0; then + AC_MSG_RESULT(yes) + eval AS_TR_CPP(HAVE_$2)=yes + else + AC_MSG_RESULT(no) + eval AS_TR_CPP(HAVE_$2)=no + fi + fi +]) From 8643e25579df7b972a91cdab3164116e185e42e7 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 5/9] BUILD: Change value of variable HAVE_PYTHON2/3_BINDINGS It will simplify detection in following patches --- diff --git a/configure.ac b/configure.ac index 64ca2c0..53a8c6b 100644 --- a/configure.ac +++ b/configure.ac @@ -322,7 +322,7 @@ AC_CHECK_PROG(HAVE_PYTHON3, python3, yes, no) AS_IF([test x$HAVE_PYTHON3 = xyes], [AC_PATH_PROG(PYTHON3, python3)]) -if test x$HAVE_PYTHON2_BINDINGS = x1; then +if test x$HAVE_PYTHON2_BINDINGS = xyes; then AS_IF([test x$HAVE_PYTHON2 != xyes], [AC_MSG_ERROR([ The program python2 was not found in search path. @@ -348,7 +348,7 @@ them please use argument --without-python2-bindings when running configure.])]) SSS_CLEAN_PYTHON_VARIABLES fi -if test x$HAVE_PYTHON3_BINDINGS = x1; then +if test x$HAVE_PYTHON3_BINDINGS = xyes; then AS_IF([test x$HAVE_PYTHON3 != xyes], [AC_MSG_ERROR([ The program python3 was not found in search path. @@ -374,7 +374,7 @@ them please use argument --without-python3-bindings when running configure.])]) SSS_CLEAN_PYTHON_VARIABLES fi -if test x$HAVE_PYTHON3_BINDINGS = x1; then +if test x$HAVE_PYTHON3_BINDINGS = xyes; then PYTHON_EXEC=$PYTHON3 else PYTHON_EXEC=$PYTHON2 diff --git a/src/conf_macros.m4 b/src/conf_macros.m4 index 5f28c78..ff9f599 100644 --- a/src/conf_macros.m4 +++ b/src/conf_macros.m4 @@ -383,7 +383,7 @@ AC_DEFUN([WITH_PYTHON2_BINDINGS], [with_python2_bindings=yes] ) if test x"$with_python2_bindings" = xyes; then - AC_SUBST([HAVE_PYTHON2_BINDINGS], [1]) + AC_SUBST([HAVE_PYTHON2_BINDINGS], [yes]) AC_DEFINE_UNQUOTED([HAVE_PYTHON2_BINDINGS], [1], [Build with python2 bindings]) fi @@ -400,7 +400,7 @@ AC_DEFUN([WITH_PYTHON3_BINDINGS], [with_python3_bindings=yes] ) if test x"$with_python3_bindings" = xyes; then - AC_SUBST([HAVE_PYTHON3_BINDINGS], [1]) + AC_SUBST([HAVE_PYTHON3_BINDINGS], [yes]) AC_DEFINE_UNQUOTED([HAVE_PYTHON3_BINDINGS], [1], [Build with python3 bindings]) fi From d906db77932b1bfaca9c29db74f05522eb7a59b4 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 6/9] BUILD: Move python checks for intgcheck to macro --- diff --git a/src/external/intgcheck.m4 b/src/external/intgcheck.m4 index 325e8ea..d358288 100644 --- a/src/external/intgcheck.m4 +++ b/src/external/intgcheck.m4 @@ -1,11 +1,5 @@ AC_CHECK_PROG([HAVE_FAKEROOT], [fakeroot], [yes], [no]) -dnl Check for pytest binary. When available, we will use py.test-2 for python2 -dnl version. If it is not available we will try to use py.test. -AC_PATH_PROG([PYTEST], [py.test]) -AC_PATH_PROG([PYTEST2], [py.test-2], [$PYTEST]) -AS_IF([test -n "$PYTEST2"], [HAVE_PYTEST2=yes], [HAVE_PYTEST2=no]) - dnl Check for variable and fail unless value is "yes" dnl The second argument will be printed in error message in case of error dnl Usage: @@ -16,6 +10,62 @@ AC_DEFUN([SSS_INTGCHECK_REQ], [ AC_MSG_ERROR([cannot enable integration tests: $2 not found])]) ]) +dnl Check for python variable and fail/warn unless value is "yes" +dnl The second argument will be printed in error message in case of error +dnl Any value in 3rd argument will make change error to info +dnl Usage: +dnl SSS_INTGCHECK_PYTHON_REQ(variable, message, [non_fatal]) + +AC_DEFUN([SSS_INTGCHECK_PYTHON_REQ], [ + AS_IF([test x$$1 = xyes], [], + [sss_have_py_intg_deps=no + AS_IF([test -n "$3"], + [AC_MSG_NOTICE([missing python dependency for integration tests: $2 not found])], + [AC_MSG_ERROR([cannot enable integration tests: $2 not found])]) + ]) +]) + +dnl Check for variable and fail unless value is "yes" +dnl The second argument will be printed in error message in case of error +dnl Usage: +dnl SSS_CHECK_PYTHON_INTG_REQ(python_version, [non_fatal]) +AC_DEFUN([SSS_CHECK_PYTHON_INTG_REQ], [ + sss_have_py_intg_deps="no" + + SSS_INTGCHECK_PYTHON_REQ([HAVE_PYTHON$1_BINDINGS], + [sssd python$1 bindings], [$2]) + + AS_IF([test x$HAVE_PYTHON$1_BINDINGS = xyes], + [SSS_CHECK_PYTEST([$PYTHON$1], [PY$1_PYTEST]) + []AM_PYTHON$1_MODULE([ldap]) + []AM_PYTHON$1_MODULE([ldb]) + []AM_PYTHON$1_MODULE([requests]) + []AM_PYTHON$1_MODULE([dbus]) + []AM_PYTHON$1_MODULE([psutil]) + + sss_have_py_intg_deps="yes" + + SSS_INTGCHECK_PYTHON_REQ([HAVE_PY$1_PYTEST], + [python$1 pytest], [$2]) + SSS_INTGCHECK_PYTHON_REQ([HAVE_PY$1MOD_LDAP], + [python$1 module ldap], [$2]) + SSS_INTGCHECK_PYTHON_REQ([HAVE_PY$1MOD_LDB], + [python$1 module ldb], [$2]) + SSS_INTGCHECK_PYTHON_REQ([HAVE_PY$1MOD_REQUESTS], + [python$1 module requests], [$2]) + SSS_INTGCHECK_PYTHON_REQ([HAVE_PY$1MOD_DBUS], + [python$1 module dbus], [$2]) + SSS_INTGCHECK_PYTHON_REQ([HAVE_PY$1MOD_PSUTIL], + [python$1 module psutil], [$2])]) + + AS_IF([test "x$sss_have_py_intg_deps" = xyes], + [HAVE_PYTHON_INTG_DEPS=yes + PYTHON_EXEC_INTG=$PYTHON$1 + AC_SUBST(PYTHON_EXEC_INTG)], + [HAVE_PYTHON_INTG_DEPS=no]) + unset sss_have_py_intg_deps +]) + AC_DEFUN([SSS_ENABLE_INTGCHECK_REQS], [ AC_ARG_ENABLE(intgcheck-reqs, [AS_HELP_STRING([--enable-intgcheck-reqs], @@ -30,17 +80,7 @@ AC_DEFUN([SSS_ENABLE_INTGCHECK_REQS], [ SSS_INTGCHECK_REQ([HAVE_LDAPMODIFY], [ldapmodify]) SSS_INTGCHECK_REQ([HAVE_FAKEROOT], [fakeroot]) - SSS_CHECK_PYTEST([$PYTHON2], [PY2_PYTEST]) - AM_PYTHON2_MODULE([ldap]) - AM_PYTHON2_MODULE([ldb]) - AM_PYTHON2_MODULE([requests]) - AM_PYTHON2_MODULE([dbus]) - AM_PYTHON2_MODULE([psutil]) - - SSS_INTGCHECK_REQ([HAVE_PYTHON2], [python2]) - SSS_INTGCHECK_REQ([HAVE_PYTEST2], [pytest2]) - SSS_INTGCHECK_REQ([HAVE_PY2MOD_LDAP], [python-ldap]) - SSS_INTGCHECK_REQ([HAVE_PY2MOD_LDB], [pyldb]) + SSS_CHECK_PYTHON_INTG_REQ([2]) fi ]) From 76a4fffd8aade1bbce1388fe980e0e67159a3b27 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 22:56:52 +0000 Subject: [PATCH 7/9] INTG: Do hot hardcode version of python/pytest in intgcheck --- diff --git a/src/tests/intg/Makefile.am b/src/tests/intg/Makefile.am index f60751a..51c90fa 100644 --- a/src/tests/intg/Makefile.am +++ b/src/tests/intg/Makefile.am @@ -191,5 +191,5 @@ intgcheck-installed: config.py passwd group pam_sss_service pam_sss_alt_service DBUS_SESSION_BUS_ADDRESS="unix:path=$$DBUS_SOCK_DIR/fake_socket" \ DBUS_SYSTEM_BUS_ADDRESS="unix:path=$$DBUS_SOCK_DIR/system_bus_socket" \ DBUS_SYSTEM_BUS_DEFAULT_ADDRESS="$$DBUS_SYSTEM_BUS_ADDRESS" \ - fakeroot $(PYTHON2) $(PYTEST2) -v -r a --tb=native $(INTGCHECK_PYTEST_ARGS) . + fakeroot $(PYTHON_EXEC_INTG) -m pytest -v -r a --tb=native $(INTGCHECK_PYTEST_ARGS) . rm -f $(DESTDIR)$(logpath)/* From 08e1b64a188e39f3f7d056bfbd63413b1a1e2929 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 23:03:31 +0000 Subject: [PATCH 8/9] BUILD: Prefer python3 for intgcheck Samba 4.11.0 dropped support for python3 and thus it was also dropped from samba related libraries (ldb ...) which is required by integration tests Merges: https://pagure.io/SSSD/sssd/pull-request/4075 Resolves: https://pagure.io/SSSD/sssd/issue/4074 --- diff --git a/src/external/intgcheck.m4 b/src/external/intgcheck.m4 index d358288..d73c5df 100644 --- a/src/external/intgcheck.m4 +++ b/src/external/intgcheck.m4 @@ -80,7 +80,12 @@ AC_DEFUN([SSS_ENABLE_INTGCHECK_REQS], [ SSS_INTGCHECK_REQ([HAVE_LDAPMODIFY], [ldapmodify]) SSS_INTGCHECK_REQ([HAVE_FAKEROOT], [fakeroot]) - SSS_CHECK_PYTHON_INTG_REQ([2]) + SSS_CHECK_PYTHON_INTG_REQ([3], [just_warning]) + + AS_IF([test "x$HAVE_PYTHON_INTG_DEPS" = xyes], [], + dnl fallback to python2 checks due to missing + dnl python3 dependencies for intgcheck + [SSS_CHECK_PYTHON_INTG_REQ([2])]) fi ]) From 05085fb7bdb50ce473acc49fa3348f4359219df6 Mon Sep 17 00:00:00 2001 From: Lukas Slebodnik Date: Sep 02 2019 23:03:56 +0000 Subject: [PATCH 9/9] intg: Install python3 dependencies for intgcheck on new distros Samba 4.11.0 dropped support for python3 and thus it was also dropped from samba related libraries (ldb ...) which is required by integration tests Merges: https://pagure.io/SSSD/sssd/pull-request/4075 Resolves: https://pagure.io/SSSD/sssd/issue/4074 --- diff --git a/contrib/ci/deps.sh b/contrib/ci/deps.sh index a5816a9..746a02a 100644 --- a/contrib/ci/deps.sh +++ b/contrib/ci/deps.sh @@ -40,10 +40,6 @@ if [[ "$DISTRO_BRANCH" == -redhat-* ]]; then nss_wrapper openldap-clients openldap-servers - pytest - python-ldap - python-psutil - pyldb rpm-build uid_wrapper pam_wrapper @@ -51,16 +47,29 @@ if [[ "$DISTRO_BRANCH" == -redhat-* ]]; then curl-devel krb5-server krb5-workstation - dbus-python ) - if [[ "$DISTRO_BRANCH" == -redhat-fedora-3[1-9]* ]]; then + if [[ "$DISTRO_BRANCH" == -redhat-fedora-3[1-9]* || + "$DISTRO_BRANCH" == -redhat-redhatenterprise*-8.*- || + "$DISTRO_BRANCH" == -redhat-centos-8.*- ]]; then DEPS_LIST+=( + python3-dbus + python3-ldap + python3-ldb + python3-psutil python3-pycodestyle + python3-pytest + python3-requests ) else DEPS_LIST+=( + dbus-python + pyldb + pytest + python-ldap python-pep8 + python-psutil + python-requests ) fi