main.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893
  1. """Core implementation of the testing process: init, session, runtest loop."""
  2. import argparse
  3. import fnmatch
  4. import functools
  5. import importlib
  6. import os
  7. import sys
  8. from pathlib import Path
  9. from typing import Callable
  10. from typing import Dict
  11. from typing import FrozenSet
  12. from typing import Iterator
  13. from typing import List
  14. from typing import Optional
  15. from typing import overload
  16. from typing import Sequence
  17. from typing import Set
  18. from typing import Tuple
  19. from typing import Type
  20. from typing import TYPE_CHECKING
  21. from typing import Union
  22. import attr
  23. import _pytest._code
  24. from _pytest import nodes
  25. from _pytest.compat import final
  26. from _pytest.config import Config
  27. from _pytest.config import directory_arg
  28. from _pytest.config import ExitCode
  29. from _pytest.config import hookimpl
  30. from _pytest.config import PytestPluginManager
  31. from _pytest.config import UsageError
  32. from _pytest.config.argparsing import Parser
  33. from _pytest.fixtures import FixtureManager
  34. from _pytest.outcomes import exit
  35. from _pytest.pathlib import absolutepath
  36. from _pytest.pathlib import bestrelpath
  37. from _pytest.pathlib import fnmatch_ex
  38. from _pytest.pathlib import visit
  39. from _pytest.reports import CollectReport
  40. from _pytest.reports import TestReport
  41. from _pytest.runner import collect_one_node
  42. from _pytest.runner import SetupState
  43. if TYPE_CHECKING:
  44. from typing_extensions import Literal
  45. def pytest_addoption(parser: Parser) -> None:
  46. parser.addini(
  47. "norecursedirs",
  48. "directory patterns to avoid for recursion",
  49. type="args",
  50. default=[
  51. "*.egg",
  52. ".*",
  53. "_darcs",
  54. "build",
  55. "CVS",
  56. "dist",
  57. "node_modules",
  58. "venv",
  59. "{arch}",
  60. ],
  61. )
  62. parser.addini(
  63. "testpaths",
  64. "directories to search for tests when no files or directories are given in the "
  65. "command line.",
  66. type="args",
  67. default=[],
  68. )
  69. group = parser.getgroup("general", "running and selection options")
  70. group._addoption(
  71. "-x",
  72. "--exitfirst",
  73. action="store_const",
  74. dest="maxfail",
  75. const=1,
  76. help="exit instantly on first error or failed test.",
  77. )
  78. group = parser.getgroup("pytest-warnings")
  79. group.addoption(
  80. "-W",
  81. "--pythonwarnings",
  82. action="append",
  83. help="set which warnings to report, see -W option of python itself.",
  84. )
  85. parser.addini(
  86. "filterwarnings",
  87. type="linelist",
  88. help="Each line specifies a pattern for "
  89. "warnings.filterwarnings. "
  90. "Processed after -W/--pythonwarnings.",
  91. )
  92. group._addoption(
  93. "--maxfail",
  94. metavar="num",
  95. action="store",
  96. type=int,
  97. dest="maxfail",
  98. default=0,
  99. help="exit after first num failures or errors.",
  100. )
  101. group._addoption(
  102. "--strict-config",
  103. action="store_true",
  104. help="any warnings encountered while parsing the `pytest` section of the configuration file raise errors.",
  105. )
  106. group._addoption(
  107. "--strict-markers",
  108. action="store_true",
  109. help="markers not registered in the `markers` section of the configuration file raise errors.",
  110. )
  111. group._addoption(
  112. "--strict",
  113. action="store_true",
  114. help="(deprecated) alias to --strict-markers.",
  115. )
  116. group._addoption(
  117. "-c",
  118. metavar="file",
  119. type=str,
  120. dest="inifilename",
  121. help="load configuration from `file` instead of trying to locate one of the implicit "
  122. "configuration files.",
  123. )
  124. group._addoption(
  125. "--continue-on-collection-errors",
  126. action="store_true",
  127. default=False,
  128. dest="continue_on_collection_errors",
  129. help="Force test execution even if collection errors occur.",
  130. )
  131. group._addoption(
  132. "--rootdir",
  133. action="store",
  134. dest="rootdir",
  135. help="Define root directory for tests. Can be relative path: 'root_dir', './root_dir', "
  136. "'root_dir/another_dir/'; absolute path: '/home/user/root_dir'; path with variables: "
  137. "'$HOME/root_dir'.",
  138. )
  139. group = parser.getgroup("collect", "collection")
  140. group.addoption(
  141. "--collectonly",
  142. "--collect-only",
  143. "--co",
  144. action="store_true",
  145. help="only collect tests, don't execute them.",
  146. )
  147. group.addoption(
  148. "--pyargs",
  149. action="store_true",
  150. help="try to interpret all arguments as python packages.",
  151. )
  152. group.addoption(
  153. "--ignore",
  154. action="append",
  155. metavar="path",
  156. help="ignore path during collection (multi-allowed).",
  157. )
  158. group.addoption(
  159. "--ignore-glob",
  160. action="append",
  161. metavar="path",
  162. help="ignore path pattern during collection (multi-allowed).",
  163. )
  164. group.addoption(
  165. "--deselect",
  166. action="append",
  167. metavar="nodeid_prefix",
  168. help="deselect item (via node id prefix) during collection (multi-allowed).",
  169. )
  170. group.addoption(
  171. "--confcutdir",
  172. dest="confcutdir",
  173. default=None,
  174. metavar="dir",
  175. type=functools.partial(directory_arg, optname="--confcutdir"),
  176. help="only load conftest.py's relative to specified dir.",
  177. )
  178. group.addoption(
  179. "--noconftest",
  180. action="store_true",
  181. dest="noconftest",
  182. default=False,
  183. help="Don't load any conftest.py files.",
  184. )
  185. group.addoption(
  186. "--keepduplicates",
  187. "--keep-duplicates",
  188. action="store_true",
  189. dest="keepduplicates",
  190. default=False,
  191. help="Keep duplicate tests.",
  192. )
  193. group.addoption(
  194. "--collect-in-virtualenv",
  195. action="store_true",
  196. dest="collect_in_virtualenv",
  197. default=False,
  198. help="Don't ignore tests in a local virtualenv directory",
  199. )
  200. group.addoption(
  201. "--import-mode",
  202. default="prepend",
  203. choices=["prepend", "append", "importlib"],
  204. dest="importmode",
  205. help="prepend/append to sys.path when importing test modules and conftest files, "
  206. "default is to prepend.",
  207. )
  208. group = parser.getgroup("debugconfig", "test session debugging and configuration")
  209. group.addoption(
  210. "--basetemp",
  211. dest="basetemp",
  212. default=None,
  213. type=validate_basetemp,
  214. metavar="dir",
  215. help=(
  216. "base temporary directory for this test run."
  217. "(warning: this directory is removed if it exists)"
  218. ),
  219. )
  220. def validate_basetemp(path: str) -> str:
  221. # GH 7119
  222. msg = "basetemp must not be empty, the current working directory or any parent directory of it"
  223. # empty path
  224. if not path:
  225. raise argparse.ArgumentTypeError(msg)
  226. def is_ancestor(base: Path, query: Path) -> bool:
  227. """Return whether query is an ancestor of base."""
  228. if base == query:
  229. return True
  230. return query in base.parents
  231. # check if path is an ancestor of cwd
  232. if is_ancestor(Path.cwd(), Path(path).absolute()):
  233. raise argparse.ArgumentTypeError(msg)
  234. # check symlinks for ancestors
  235. if is_ancestor(Path.cwd().resolve(), Path(path).resolve()):
  236. raise argparse.ArgumentTypeError(msg)
  237. return path
  238. def wrap_session(
  239. config: Config, doit: Callable[[Config, "Session"], Optional[Union[int, ExitCode]]]
  240. ) -> Union[int, ExitCode]:
  241. """Skeleton command line program."""
  242. session = Session.from_config(config)
  243. session.exitstatus = ExitCode.OK
  244. initstate = 0
  245. try:
  246. try:
  247. config._do_configure()
  248. initstate = 1
  249. config.hook.pytest_sessionstart(session=session)
  250. initstate = 2
  251. session.exitstatus = doit(config, session) or 0
  252. except UsageError:
  253. session.exitstatus = ExitCode.USAGE_ERROR
  254. raise
  255. except Failed:
  256. session.exitstatus = ExitCode.TESTS_FAILED
  257. except (KeyboardInterrupt, exit.Exception):
  258. excinfo = _pytest._code.ExceptionInfo.from_current()
  259. exitstatus: Union[int, ExitCode] = ExitCode.INTERRUPTED
  260. if isinstance(excinfo.value, exit.Exception):
  261. if excinfo.value.returncode is not None:
  262. exitstatus = excinfo.value.returncode
  263. if initstate < 2:
  264. sys.stderr.write(f"{excinfo.typename}: {excinfo.value.msg}\n")
  265. config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
  266. session.exitstatus = exitstatus
  267. except BaseException:
  268. session.exitstatus = ExitCode.INTERNAL_ERROR
  269. excinfo = _pytest._code.ExceptionInfo.from_current()
  270. try:
  271. config.notify_exception(excinfo, config.option)
  272. except exit.Exception as exc:
  273. if exc.returncode is not None:
  274. session.exitstatus = exc.returncode
  275. sys.stderr.write(f"{type(exc).__name__}: {exc}\n")
  276. else:
  277. if isinstance(excinfo.value, SystemExit):
  278. sys.stderr.write("mainloop: caught unexpected SystemExit!\n")
  279. finally:
  280. # Explicitly break reference cycle.
  281. excinfo = None # type: ignore
  282. os.chdir(session.startpath)
  283. if initstate >= 2:
  284. try:
  285. config.hook.pytest_sessionfinish(
  286. session=session, exitstatus=session.exitstatus
  287. )
  288. except exit.Exception as exc:
  289. if exc.returncode is not None:
  290. session.exitstatus = exc.returncode
  291. sys.stderr.write(f"{type(exc).__name__}: {exc}\n")
  292. config._ensure_unconfigure()
  293. return session.exitstatus
  294. def pytest_cmdline_main(config: Config) -> Union[int, ExitCode]:
  295. return wrap_session(config, _main)
  296. def _main(config: Config, session: "Session") -> Optional[Union[int, ExitCode]]:
  297. """Default command line protocol for initialization, session,
  298. running tests and reporting."""
  299. config.hook.pytest_collection(session=session)
  300. config.hook.pytest_runtestloop(session=session)
  301. if session.testsfailed:
  302. return ExitCode.TESTS_FAILED
  303. elif session.testscollected == 0:
  304. return ExitCode.NO_TESTS_COLLECTED
  305. return None
  306. def pytest_collection(session: "Session") -> None:
  307. session.perform_collect()
  308. def pytest_runtestloop(session: "Session") -> bool:
  309. if session.testsfailed and not session.config.option.continue_on_collection_errors:
  310. raise session.Interrupted(
  311. "%d error%s during collection"
  312. % (session.testsfailed, "s" if session.testsfailed != 1 else "")
  313. )
  314. if session.config.option.collectonly:
  315. return True
  316. for i, item in enumerate(session.items):
  317. nextitem = session.items[i + 1] if i + 1 < len(session.items) else None
  318. item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
  319. if session.shouldfail:
  320. raise session.Failed(session.shouldfail)
  321. if session.shouldstop:
  322. raise session.Interrupted(session.shouldstop)
  323. return True
  324. def _in_venv(path: Path) -> bool:
  325. """Attempt to detect if ``path`` is the root of a Virtual Environment by
  326. checking for the existence of the appropriate activate script."""
  327. bindir = path.joinpath("Scripts" if sys.platform.startswith("win") else "bin")
  328. try:
  329. if not bindir.is_dir():
  330. return False
  331. except OSError:
  332. return False
  333. activates = (
  334. "activate",
  335. "activate.csh",
  336. "activate.fish",
  337. "Activate",
  338. "Activate.bat",
  339. "Activate.ps1",
  340. )
  341. return any(fname.name in activates for fname in bindir.iterdir())
  342. def pytest_ignore_collect(collection_path: Path, config: Config) -> Optional[bool]:
  343. ignore_paths = config._getconftest_pathlist(
  344. "collect_ignore", path=collection_path.parent, rootpath=config.rootpath
  345. )
  346. ignore_paths = ignore_paths or []
  347. excludeopt = config.getoption("ignore")
  348. if excludeopt:
  349. ignore_paths.extend(absolutepath(x) for x in excludeopt)
  350. if collection_path in ignore_paths:
  351. return True
  352. ignore_globs = config._getconftest_pathlist(
  353. "collect_ignore_glob", path=collection_path.parent, rootpath=config.rootpath
  354. )
  355. ignore_globs = ignore_globs or []
  356. excludeglobopt = config.getoption("ignore_glob")
  357. if excludeglobopt:
  358. ignore_globs.extend(absolutepath(x) for x in excludeglobopt)
  359. if any(fnmatch.fnmatch(str(collection_path), str(glob)) for glob in ignore_globs):
  360. return True
  361. allow_in_venv = config.getoption("collect_in_virtualenv")
  362. if not allow_in_venv and _in_venv(collection_path):
  363. return True
  364. return None
  365. def pytest_collection_modifyitems(items: List[nodes.Item], config: Config) -> None:
  366. deselect_prefixes = tuple(config.getoption("deselect") or [])
  367. if not deselect_prefixes:
  368. return
  369. remaining = []
  370. deselected = []
  371. for colitem in items:
  372. if colitem.nodeid.startswith(deselect_prefixes):
  373. deselected.append(colitem)
  374. else:
  375. remaining.append(colitem)
  376. if deselected:
  377. config.hook.pytest_deselected(items=deselected)
  378. items[:] = remaining
  379. class FSHookProxy:
  380. def __init__(self, pm: PytestPluginManager, remove_mods) -> None:
  381. self.pm = pm
  382. self.remove_mods = remove_mods
  383. def __getattr__(self, name: str):
  384. x = self.pm.subset_hook_caller(name, remove_plugins=self.remove_mods)
  385. self.__dict__[name] = x
  386. return x
  387. class Interrupted(KeyboardInterrupt):
  388. """Signals that the test run was interrupted."""
  389. __module__ = "builtins" # For py3.
  390. class Failed(Exception):
  391. """Signals a stop as failed test run."""
  392. @attr.s(slots=True, auto_attribs=True)
  393. class _bestrelpath_cache(Dict[Path, str]):
  394. path: Path
  395. def __missing__(self, path: Path) -> str:
  396. r = bestrelpath(self.path, path)
  397. self[path] = r
  398. return r
  399. @final
  400. class Session(nodes.FSCollector):
  401. Interrupted = Interrupted
  402. Failed = Failed
  403. # Set on the session by runner.pytest_sessionstart.
  404. _setupstate: SetupState
  405. # Set on the session by fixtures.pytest_sessionstart.
  406. _fixturemanager: FixtureManager
  407. exitstatus: Union[int, ExitCode]
  408. def __init__(self, config: Config) -> None:
  409. super().__init__(
  410. path=config.rootpath,
  411. fspath=None,
  412. parent=None,
  413. config=config,
  414. session=self,
  415. nodeid="",
  416. )
  417. self.testsfailed = 0
  418. self.testscollected = 0
  419. self.shouldstop: Union[bool, str] = False
  420. self.shouldfail: Union[bool, str] = False
  421. self.trace = config.trace.root.get("collection")
  422. self._initialpaths: FrozenSet[Path] = frozenset()
  423. self._bestrelpathcache: Dict[Path, str] = _bestrelpath_cache(config.rootpath)
  424. self.config.pluginmanager.register(self, name="session")
  425. @classmethod
  426. def from_config(cls, config: Config) -> "Session":
  427. session: Session = cls._create(config=config)
  428. return session
  429. def __repr__(self) -> str:
  430. return "<%s %s exitstatus=%r testsfailed=%d testscollected=%d>" % (
  431. self.__class__.__name__,
  432. self.name,
  433. getattr(self, "exitstatus", "<UNSET>"),
  434. self.testsfailed,
  435. self.testscollected,
  436. )
  437. @property
  438. def startpath(self) -> Path:
  439. """The path from which pytest was invoked.
  440. .. versionadded:: 7.0.0
  441. """
  442. return self.config.invocation_params.dir
  443. def _node_location_to_relpath(self, node_path: Path) -> str:
  444. # bestrelpath is a quite slow function.
  445. return self._bestrelpathcache[node_path]
  446. @hookimpl(tryfirst=True)
  447. def pytest_collectstart(self) -> None:
  448. if self.shouldfail:
  449. raise self.Failed(self.shouldfail)
  450. if self.shouldstop:
  451. raise self.Interrupted(self.shouldstop)
  452. @hookimpl(tryfirst=True)
  453. def pytest_runtest_logreport(
  454. self, report: Union[TestReport, CollectReport]
  455. ) -> None:
  456. if report.failed and not hasattr(report, "wasxfail"):
  457. self.testsfailed += 1
  458. maxfail = self.config.getvalue("maxfail")
  459. if maxfail and self.testsfailed >= maxfail:
  460. self.shouldfail = "stopping after %d failures" % (self.testsfailed)
  461. pytest_collectreport = pytest_runtest_logreport
  462. def isinitpath(self, path: Union[str, "os.PathLike[str]"]) -> bool:
  463. # Optimization: Path(Path(...)) is much slower than isinstance.
  464. path_ = path if isinstance(path, Path) else Path(path)
  465. return path_ in self._initialpaths
  466. def gethookproxy(self, fspath: "os.PathLike[str]"):
  467. # Optimization: Path(Path(...)) is much slower than isinstance.
  468. path = fspath if isinstance(fspath, Path) else Path(fspath)
  469. pm = self.config.pluginmanager
  470. # Check if we have the common case of running
  471. # hooks with all conftest.py files.
  472. my_conftestmodules = pm._getconftestmodules(
  473. path,
  474. self.config.getoption("importmode"),
  475. rootpath=self.config.rootpath,
  476. )
  477. remove_mods = pm._conftest_plugins.difference(my_conftestmodules)
  478. if remove_mods:
  479. # One or more conftests are not in use at this fspath.
  480. from .config.compat import PathAwareHookProxy
  481. proxy = PathAwareHookProxy(FSHookProxy(pm, remove_mods))
  482. else:
  483. # All plugins are active for this fspath.
  484. proxy = self.config.hook
  485. return proxy
  486. def _recurse(self, direntry: "os.DirEntry[str]") -> bool:
  487. if direntry.name == "__pycache__":
  488. return False
  489. fspath = Path(direntry.path)
  490. ihook = self.gethookproxy(fspath.parent)
  491. if ihook.pytest_ignore_collect(collection_path=fspath, config=self.config):
  492. return False
  493. norecursepatterns = self.config.getini("norecursedirs")
  494. if any(fnmatch_ex(pat, fspath) for pat in norecursepatterns):
  495. return False
  496. return True
  497. def _collectfile(
  498. self, fspath: Path, handle_dupes: bool = True
  499. ) -> Sequence[nodes.Collector]:
  500. assert (
  501. fspath.is_file()
  502. ), "{!r} is not a file (isdir={!r}, exists={!r}, islink={!r})".format(
  503. fspath, fspath.is_dir(), fspath.exists(), fspath.is_symlink()
  504. )
  505. ihook = self.gethookproxy(fspath)
  506. if not self.isinitpath(fspath):
  507. if ihook.pytest_ignore_collect(collection_path=fspath, config=self.config):
  508. return ()
  509. if handle_dupes:
  510. keepduplicates = self.config.getoption("keepduplicates")
  511. if not keepduplicates:
  512. duplicate_paths = self.config.pluginmanager._duplicatepaths
  513. if fspath in duplicate_paths:
  514. return ()
  515. else:
  516. duplicate_paths.add(fspath)
  517. return ihook.pytest_collect_file(file_path=fspath, parent=self) # type: ignore[no-any-return]
  518. @overload
  519. def perform_collect(
  520. self, args: Optional[Sequence[str]] = ..., genitems: "Literal[True]" = ...
  521. ) -> Sequence[nodes.Item]:
  522. ...
  523. @overload
  524. def perform_collect(
  525. self, args: Optional[Sequence[str]] = ..., genitems: bool = ...
  526. ) -> Sequence[Union[nodes.Item, nodes.Collector]]:
  527. ...
  528. def perform_collect(
  529. self, args: Optional[Sequence[str]] = None, genitems: bool = True
  530. ) -> Sequence[Union[nodes.Item, nodes.Collector]]:
  531. """Perform the collection phase for this session.
  532. This is called by the default :hook:`pytest_collection` hook
  533. implementation; see the documentation of this hook for more details.
  534. For testing purposes, it may also be called directly on a fresh
  535. ``Session``.
  536. This function normally recursively expands any collectors collected
  537. from the session to their items, and only items are returned. For
  538. testing purposes, this may be suppressed by passing ``genitems=False``,
  539. in which case the return value contains these collectors unexpanded,
  540. and ``session.items`` is empty.
  541. """
  542. if args is None:
  543. args = self.config.args
  544. self.trace("perform_collect", self, args)
  545. self.trace.root.indent += 1
  546. self._notfound: List[Tuple[str, Sequence[nodes.Collector]]] = []
  547. self._initial_parts: List[Tuple[Path, List[str]]] = []
  548. self.items: List[nodes.Item] = []
  549. hook = self.config.hook
  550. items: Sequence[Union[nodes.Item, nodes.Collector]] = self.items
  551. try:
  552. initialpaths: List[Path] = []
  553. for arg in args:
  554. fspath, parts = resolve_collection_argument(
  555. self.config.invocation_params.dir,
  556. arg,
  557. as_pypath=self.config.option.pyargs,
  558. )
  559. self._initial_parts.append((fspath, parts))
  560. initialpaths.append(fspath)
  561. self._initialpaths = frozenset(initialpaths)
  562. rep = collect_one_node(self)
  563. self.ihook.pytest_collectreport(report=rep)
  564. self.trace.root.indent -= 1
  565. if self._notfound:
  566. errors = []
  567. for arg, cols in self._notfound:
  568. line = f"(no name {arg!r} in any of {cols!r})"
  569. errors.append(f"not found: {arg}\n{line}")
  570. raise UsageError(*errors)
  571. if not genitems:
  572. items = rep.result
  573. else:
  574. if rep.passed:
  575. for node in rep.result:
  576. self.items.extend(self.genitems(node))
  577. self.config.pluginmanager.check_pending()
  578. hook.pytest_collection_modifyitems(
  579. session=self, config=self.config, items=items
  580. )
  581. finally:
  582. hook.pytest_collection_finish(session=self)
  583. self.testscollected = len(items)
  584. return items
  585. def collect(self) -> Iterator[Union[nodes.Item, nodes.Collector]]:
  586. from _pytest.python import Package
  587. # Keep track of any collected nodes in here, so we don't duplicate fixtures.
  588. node_cache1: Dict[Path, Sequence[nodes.Collector]] = {}
  589. node_cache2: Dict[Tuple[Type[nodes.Collector], Path], nodes.Collector] = {}
  590. # Keep track of any collected collectors in matchnodes paths, so they
  591. # are not collected more than once.
  592. matchnodes_cache: Dict[Tuple[Type[nodes.Collector], str], CollectReport] = {}
  593. # Dirnames of pkgs with dunder-init files.
  594. pkg_roots: Dict[str, Package] = {}
  595. for argpath, names in self._initial_parts:
  596. self.trace("processing argument", (argpath, names))
  597. self.trace.root.indent += 1
  598. # Start with a Session root, and delve to argpath item (dir or file)
  599. # and stack all Packages found on the way.
  600. # No point in finding packages when collecting doctests.
  601. if not self.config.getoption("doctestmodules", False):
  602. pm = self.config.pluginmanager
  603. confcutdir = pm._confcutdir
  604. for parent in (argpath, *argpath.parents):
  605. if confcutdir and parent in confcutdir.parents:
  606. break
  607. if parent.is_dir():
  608. pkginit = parent / "__init__.py"
  609. if pkginit.is_file() and pkginit not in node_cache1:
  610. col = self._collectfile(pkginit, handle_dupes=False)
  611. if col:
  612. if isinstance(col[0], Package):
  613. pkg_roots[str(parent)] = col[0]
  614. node_cache1[col[0].path] = [col[0]]
  615. # If it's a directory argument, recurse and look for any Subpackages.
  616. # Let the Package collector deal with subnodes, don't collect here.
  617. if argpath.is_dir():
  618. assert not names, f"invalid arg {(argpath, names)!r}"
  619. seen_dirs: Set[Path] = set()
  620. for direntry in visit(str(argpath), self._recurse):
  621. if not direntry.is_file():
  622. continue
  623. path = Path(direntry.path)
  624. dirpath = path.parent
  625. if dirpath not in seen_dirs:
  626. # Collect packages first.
  627. seen_dirs.add(dirpath)
  628. pkginit = dirpath / "__init__.py"
  629. if pkginit.exists():
  630. for x in self._collectfile(pkginit):
  631. yield x
  632. if isinstance(x, Package):
  633. pkg_roots[str(dirpath)] = x
  634. if str(dirpath) in pkg_roots:
  635. # Do not collect packages here.
  636. continue
  637. for x in self._collectfile(path):
  638. key2 = (type(x), x.path)
  639. if key2 in node_cache2:
  640. yield node_cache2[key2]
  641. else:
  642. node_cache2[key2] = x
  643. yield x
  644. else:
  645. assert argpath.is_file()
  646. if argpath in node_cache1:
  647. col = node_cache1[argpath]
  648. else:
  649. collect_root = pkg_roots.get(str(argpath.parent), self)
  650. col = collect_root._collectfile(argpath, handle_dupes=False)
  651. if col:
  652. node_cache1[argpath] = col
  653. matching = []
  654. work: List[
  655. Tuple[Sequence[Union[nodes.Item, nodes.Collector]], Sequence[str]]
  656. ] = [(col, names)]
  657. while work:
  658. self.trace("matchnodes", col, names)
  659. self.trace.root.indent += 1
  660. matchnodes, matchnames = work.pop()
  661. for node in matchnodes:
  662. if not matchnames:
  663. matching.append(node)
  664. continue
  665. if not isinstance(node, nodes.Collector):
  666. continue
  667. key = (type(node), node.nodeid)
  668. if key in matchnodes_cache:
  669. rep = matchnodes_cache[key]
  670. else:
  671. rep = collect_one_node(node)
  672. matchnodes_cache[key] = rep
  673. if rep.passed:
  674. submatchnodes = []
  675. for r in rep.result:
  676. # TODO: Remove parametrized workaround once collection structure contains
  677. # parametrization.
  678. if (
  679. r.name == matchnames[0]
  680. or r.name.split("[")[0] == matchnames[0]
  681. ):
  682. submatchnodes.append(r)
  683. if submatchnodes:
  684. work.append((submatchnodes, matchnames[1:]))
  685. else:
  686. # Report collection failures here to avoid failing to run some test
  687. # specified in the command line because the module could not be
  688. # imported (#134).
  689. node.ihook.pytest_collectreport(report=rep)
  690. self.trace("matchnodes finished -> ", len(matching), "nodes")
  691. self.trace.root.indent -= 1
  692. if not matching:
  693. report_arg = "::".join((str(argpath), *names))
  694. self._notfound.append((report_arg, col))
  695. continue
  696. # If __init__.py was the only file requested, then the matched
  697. # node will be the corresponding Package (by default), and the
  698. # first yielded item will be the __init__ Module itself, so
  699. # just use that. If this special case isn't taken, then all the
  700. # files in the package will be yielded.
  701. if argpath.name == "__init__.py" and isinstance(matching[0], Package):
  702. try:
  703. yield next(iter(matching[0].collect()))
  704. except StopIteration:
  705. # The package collects nothing with only an __init__.py
  706. # file in it, which gets ignored by the default
  707. # "python_files" option.
  708. pass
  709. continue
  710. yield from matching
  711. self.trace.root.indent -= 1
  712. def genitems(
  713. self, node: Union[nodes.Item, nodes.Collector]
  714. ) -> Iterator[nodes.Item]:
  715. self.trace("genitems", node)
  716. if isinstance(node, nodes.Item):
  717. node.ihook.pytest_itemcollected(item=node)
  718. yield node
  719. else:
  720. assert isinstance(node, nodes.Collector)
  721. rep = collect_one_node(node)
  722. if rep.passed:
  723. for subnode in rep.result:
  724. yield from self.genitems(subnode)
  725. node.ihook.pytest_collectreport(report=rep)
  726. def search_pypath(module_name: str) -> str:
  727. """Search sys.path for the given a dotted module name, and return its file system path."""
  728. try:
  729. spec = importlib.util.find_spec(module_name)
  730. # AttributeError: looks like package module, but actually filename
  731. # ImportError: module does not exist
  732. # ValueError: not a module name
  733. except (AttributeError, ImportError, ValueError):
  734. return module_name
  735. if spec is None or spec.origin is None or spec.origin == "namespace":
  736. return module_name
  737. elif spec.submodule_search_locations:
  738. return os.path.dirname(spec.origin)
  739. else:
  740. return spec.origin
  741. def resolve_collection_argument(
  742. invocation_path: Path, arg: str, *, as_pypath: bool = False
  743. ) -> Tuple[Path, List[str]]:
  744. """Parse path arguments optionally containing selection parts and return (fspath, names).
  745. Command-line arguments can point to files and/or directories, and optionally contain
  746. parts for specific tests selection, for example:
  747. "pkg/tests/test_foo.py::TestClass::test_foo"
  748. This function ensures the path exists, and returns a tuple:
  749. (Path("/full/path/to/pkg/tests/test_foo.py"), ["TestClass", "test_foo"])
  750. When as_pypath is True, expects that the command-line argument actually contains
  751. module paths instead of file-system paths:
  752. "pkg.tests.test_foo::TestClass::test_foo"
  753. In which case we search sys.path for a matching module, and then return the *path* to the
  754. found module.
  755. If the path doesn't exist, raise UsageError.
  756. If the path is a directory and selection parts are present, raise UsageError.
  757. """
  758. strpath, *parts = str(arg).split("::")
  759. if as_pypath:
  760. strpath = search_pypath(strpath)
  761. fspath = invocation_path / strpath
  762. fspath = absolutepath(fspath)
  763. if not fspath.exists():
  764. msg = (
  765. "module or package not found: {arg} (missing __init__.py?)"
  766. if as_pypath
  767. else "file or directory not found: {arg}"
  768. )
  769. raise UsageError(msg.format(arg=arg))
  770. if parts and fspath.is_dir():
  771. msg = (
  772. "package argument cannot contain :: selection parts: {arg}"
  773. if as_pypath
  774. else "directory argument cannot contain :: selection parts: {arg}"
  775. )
  776. raise UsageError(msg.format(arg=arg))
  777. return fspath, parts