build: add pytest targets

It enables running pytests in cmake jobs, regardless of underlying build
tool choice (= makes it work with ninja.)

Also:

- drop pytest logic launching `make` and exiting in case of failure.
  Maybe there is a better way and keep this functionality somehow, bind
  it to a command-line option? make it fail softly?

- GHA/linux: invoke pytest via the build, not directly.

- autotools: add missing dummy runtests targets when cross-compiling.

Closes curl#15034

Author: vszakats

.github/workflows/linux.yml

@@ -652,14 +652,17 @@ jobs:
           fi
           make V=1 VERBOSE=1 test-ci
 
-      - if: contains(matrix.build.install_steps, 'pytest')
-        # run for `tests/http` directory, so pytest does not pick up any other
-        # packages we might have built here
-        run: pytest -v tests/http
-        name: 'run pytest'
+      - name: 'run pytest'
+        if: contains(matrix.build.install_steps, 'pytest')
         env:
           TFLAGS: "${{ matrix.build.tflags }}"
           CURL_CI: github
+        run: |
+          if [ -n '${{ matrix.build.generate }}' ]; then
+            cmake --build . --verbose --target curl-pytest-ci
+          else
+            make V=1 pytest-ci
+          fi
 
       - run: ${{ matrix.build.make-prefix }} make V=1 examples
         if: ${{ matrix.build.configure }}

Makefile.am

@@ -118,7 +118,13 @@ check: test examples check-docs
 
 if CROSSCOMPILING
 test-full: test
+test-nonflaky: test
 test-torture: test
+test-event: test
+test-am: test
+test-ci: test
+pytest: test
+pytest-ci: test
 
 test:
 	@echo "NOTICE: we can't run the tests when cross-compiling!"
@@ -146,6 +152,12 @@ test-am:
 test-ci:
 	@(cd tests; $(MAKE) all ci-test)
 
+pytest:
+	@(cd tests; $(MAKE) all default-pytest)
+
+pytest-ci:
+	@(cd tests; $(MAKE) all ci-pytest)
+
 endif
 
 examples:

tests/CMakeLists.txt

@@ -43,10 +43,10 @@ function(add_runtests _targetname _test_flags)
   if(CURL_TEST_BUNDLES)
     set(_test_flags "${_test_flags} -bundle")
   endif()
-  unset(_depends)
   # Skip walking through dependent targets before running tests in CI.
   # This avoids: GNU Make doing a slow re-evaluation of all targets and
   # skipping them, MSBuild doing a re-evaluation, and actually rebuilding them.
+  unset(_depends)
   if(NOT _targetname STREQUAL "test-ci")
     set(_depends "testdeps")
   endif()
@@ -65,6 +65,20 @@ function(add_runtests _targetname _test_flags)
   )
 endfunction()
 
+function(add_pytest _targetname _test_flags)
+  unset(_depends)
+  if(NOT _targetname STREQUAL "pytest-ci")
+    set(_depends "test-http-clients")
+  endif()
+  string(REPLACE " " ";" _test_flags_list "${_test_flags}")
+  add_custom_target(${_targetname}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND pytest ${_test_flags_list} "${CMAKE_CURRENT_SOURCE_DIR}/http"
+    DEPENDS "${_depends}"
+    VERBATIM USES_TERMINAL
+  )
+endfunction()
+
 # Create configurehelp.pm, used by tests needing to run the C preprocessor.
 if(MSVC OR CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
   set(CURL_CPP "\"${CMAKE_C_COMPILER}\" -E")
@@ -94,3 +108,6 @@ add_runtests(test-nonflaky  "-a -p ~flaky ~timing-dependent")
 add_runtests(test-ci        "-a -p ~flaky ~timing-dependent -r -rm -j2")
 add_runtests(test-torture   "-a -t -j2")
 add_runtests(test-event     "-a -e")
+
+add_pytest(curl-pytest      "")
+add_pytest(curl-pytest-ci   "-v")

tests/Makefile.am

@@ -122,6 +122,7 @@ TEST_COMMON =
 
 if CROSSCOMPILING
 TEST = @echo "NOTICE: we can't run the tests when cross-compiling!"
+PYTEST = $(TEST)
 else # if not cross-compiling:
 if USE_TEST_BUNDLES
 TEST_COMMON += -bundle
@@ -140,6 +141,8 @@ TEST_NF = -a -p ~flaky ~timing-dependent
 
 # special CI target derived from nonflaky with CI-specific flags
 TEST_CI = $(TEST_NF) -r -rm -j2
+
+PYTEST = pytest
 endif
 
 CD2NROFF = $(top_srcdir)/scripts/cd2nroff $< >$@
@@ -177,6 +180,11 @@ torture-test: perlcheck all
 event-test: perlcheck all
 	$(TEST) $(TEST_E) $(TFLAGS)
 
+default-pytest: ci-pytest
+
+ci-pytest: all
+	srcdir=$(srcdir) $(PYTEST) -v $(srcdir)/http
+
 checksrc:
 	(cd libtest && $(MAKE) checksrc)
 	(cd unit && $(MAKE) checksrc)

tests/http/clients/CMakeLists.txt

@@ -22,6 +22,8 @@
 #
 ###########################################################################
 
+add_custom_target(test-http-clients)
+
 # Get 'check_PROGRAMS' variable
 transform_makefile_inc("Makefile.inc" "${CMAKE_CURRENT_BINARY_DIR}/Makefile.inc.cmake")
 include("${CMAKE_CURRENT_BINARY_DIR}/Makefile.inc.cmake")
@@ -30,6 +32,7 @@ foreach(_target IN LISTS check_PROGRAMS)
   set(_target_name "curlt-client-${_target}")
   add_executable(${_target_name} EXCLUDE_FROM_ALL "${_target}.c")
   add_dependencies(testdeps ${_target_name})
+  add_dependencies(test-http-clients ${_target_name})
   target_include_directories(${_target_name} PRIVATE
     "${CURL_BINARY_DIR}/lib"  # for "curl_config.h"
     "${CURL_SOURCE_DIR}/lib"  # for "curl_setup.h"

tests/http/conftest.py

@@ -73,8 +73,6 @@ def env(pytestconfig) -> Env:
         pytest.skip(env.incomplete_reason())
 
     env.setup()
-    if not env.make_clients():
-        pytest.exit(1)
     return env
 
 @pytest.fixture(scope="package", autouse=True)

tests/http/testenv/env.py

@@ -579,12 +579,3 @@ def make_data_file(self, indir: str, fname: str, fsize: int,
                 i = int(fsize / line_length) + 1
                 fd.write(f"{i:09d}-{s}"[0:remain-1] + "\n")
         return fpath
-
-    def make_clients(self):
-        client_dir = os.path.join(self.project_dir, 'tests/http/clients')
-        p = subprocess.run(['make'], capture_output=True, text=True,
-                           cwd=client_dir)
-        if p.returncode != 0:
-            pytest.exit(f"`make`in {client_dir} failed:\n{p.stderr}")
-            return False
-        return True