b731fff69c
Add USES=pytest, a standardized pytest entry point. This knob adds dependency on pytest and defines do-test target which calls it "the right way" (respecting TEST_ENV, with proper verbosity, additional info on skipped tests, and resetting addopts which may be set by upstream and introduce unwanted options). This allows to remove duplicate (and in many cases not completely correct) do-test targets from a lot of pytest consumer ports. Additionally, it adds PYTEST_IGNORED_TESTS (to list tests which fail and are expected to fail) and PYTEST_BROKEN_TESTS (to list tests which fail and needs to be fixed). Both knobs lead to listed tests being skipped, which is believed to be better than always failing `test` target, allowing to monitor status of other tests and better document known failures. Possible room for improvement: - Support for pytest plugins (add dependencies and disallow pytest from using unlisted plugins which often cause breakages) - Treat PYTEST_BROKEN_TESTS as xfail to catch the cases when tests are fixed. Both items seem to require support on pytest side though. Reviewed by: tcberner Differential Revision: https://reviews.freebsd.org/D34693
69 lines
2.1 KiB
Makefile
69 lines
2.1 KiB
Makefile
# handle testing with pytest
|
|
#
|
|
# Feature: pytest
|
|
# Usage: USES=pytest[:4]
|
|
#
|
|
# It implies USES=python:test automatically if no USES=python has been
|
|
# specified yet
|
|
#
|
|
# It provides the following additional variables to set by the ports:
|
|
#
|
|
# PYTEST_ARGS additional args to pytest (defaults to empty).
|
|
# PYTEST_IGNORED_TESTS lists of `pytest -k` patterns of tests to ignore
|
|
# (defaults to empty). For tests which are not
|
|
# expected to pass, such as ones requiring a database
|
|
# access.
|
|
# PYTEST_BROKEN_TESTS lists of `pytest -k` patterns of tests to ignore
|
|
# (defaults to empty). For broken tests which require
|
|
# fixing.
|
|
#
|
|
# The following variables may be set by the user:
|
|
#
|
|
# PYTEST_ENABLE_IGNORED_TESTS enable tests which are otherwise ignored by
|
|
# PYTEST_IGNORED_TESTS.
|
|
# PYTEST_ENABLE_BROKEN_TESTS enable tests which are otherwise ignored by
|
|
# PYTEST_BROKEN_TESTS.
|
|
# PYTEST_ENABLE_ALL_TESTS enable tests which are otherwise ignored by
|
|
# PYTEST_IGNORED_TESTS and PYTEST_BROKEN_TESTS.
|
|
#
|
|
# MAINTAINER: amdmi3@FreeBSD.org
|
|
|
|
.if !defined(_INCLUDE_USES_PYTEST_MK)
|
|
_INCLUDE_USES_PYTEST_MK= yes
|
|
|
|
. if !${USES:Mpython*}
|
|
python_ARGS= test
|
|
. include "${USESDIR}/python.mk"
|
|
. endif
|
|
|
|
. if empty(pytest_ARGS)
|
|
TEST_DEPENDS+= ${PYTHON_PKGNAMEPREFIX}pytest>=0:devel/py-pytest@${PY_FLAVOR}
|
|
. elif ${pytest_ARGS} == "4"
|
|
TEST_DEPENDS+= ${PYTHON_PKGNAMEPREFIX}pytest4>=0:devel/py-pytest4@${PY_FLAVOR}
|
|
. else
|
|
IGNORE= Incorrect 'USES+=pytest:${pytest_ARGS}' expecting 'USES+=pytest[:4]'
|
|
. endif
|
|
|
|
PYTEST_IGNORED_TESTS?= # empty
|
|
PYTEST_BROKEN_TESTS?= # empty
|
|
PYTEST_ARGS?= # empty
|
|
|
|
_PYTEST_ALL_IGNORED_TESTS?= # empty
|
|
. if !defined(PYTEST_ENABLE_IGNORED_TESTS) && !defined(PYTEST_ENABLE_ALL_TESTS)
|
|
_PYTEST_ALL_IGNORED_TESTS+= ${PYTEST_IGNORED_TESTS}
|
|
. endif
|
|
. if !defined(PYTEST_ENABLE_BROKEN_TESTS) && !defined(PYTEST_ENABLE_ALL_TESTS)
|
|
_PYTEST_ALL_IGNORED_TESTS+= ${PYTEST_BROKEN_TESTS}
|
|
. endif
|
|
|
|
_PYTEST_FILTER_EXPRESSION= ${_PYTEST_ALL_IGNORED_TESTS:C/^(.)/and not \1/:tW:C/^and //}
|
|
|
|
. if !target(do-test)
|
|
do-test:
|
|
@cd ${TEST_WRKSRC} && ${SETENV} ${TEST_ENV} ${PYTHON_CMD} -m pytest \
|
|
-k '${_PYTEST_FILTER_EXPRESSION}' \
|
|
-v -rs -o addopts= \
|
|
${PYTEST_ARGS}
|
|
. endif
|
|
|
|
.endif
|