req_command.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506
  1. """Contains the Command base classes that depend on PipSession.
  2. The classes in this module are in a separate module so the commands not
  3. needing download / PackageFinder capability don't unnecessarily import the
  4. PackageFinder machinery and all its vendored dependencies, etc.
  5. """
  6. import logging
  7. import os
  8. import sys
  9. from functools import partial
  10. from optparse import Values
  11. from typing import Any, List, Optional, Tuple
  12. from pip._internal.cache import WheelCache
  13. from pip._internal.cli import cmdoptions
  14. from pip._internal.cli.base_command import Command
  15. from pip._internal.cli.command_context import CommandContextMixIn
  16. from pip._internal.exceptions import CommandError, PreviousBuildDirError
  17. from pip._internal.index.collector import LinkCollector
  18. from pip._internal.index.package_finder import PackageFinder
  19. from pip._internal.models.selection_prefs import SelectionPreferences
  20. from pip._internal.models.target_python import TargetPython
  21. from pip._internal.network.session import PipSession
  22. from pip._internal.operations.prepare import RequirementPreparer
  23. from pip._internal.req.constructors import (
  24. install_req_from_editable,
  25. install_req_from_line,
  26. install_req_from_parsed_requirement,
  27. install_req_from_req_string,
  28. )
  29. from pip._internal.req.req_file import parse_requirements
  30. from pip._internal.req.req_install import InstallRequirement
  31. from pip._internal.req.req_tracker import RequirementTracker
  32. from pip._internal.resolution.base import BaseResolver
  33. from pip._internal.self_outdated_check import pip_self_version_check
  34. from pip._internal.utils.deprecation import deprecated
  35. from pip._internal.utils.temp_dir import (
  36. TempDirectory,
  37. TempDirectoryTypeRegistry,
  38. tempdir_kinds,
  39. )
  40. from pip._internal.utils.virtualenv import running_under_virtualenv
  41. logger = logging.getLogger(__name__)
  42. class SessionCommandMixin(CommandContextMixIn):
  43. """
  44. A class mixin for command classes needing _build_session().
  45. """
  46. def __init__(self) -> None:
  47. super().__init__()
  48. self._session: Optional[PipSession] = None
  49. @classmethod
  50. def _get_index_urls(cls, options: Values) -> Optional[List[str]]:
  51. """Return a list of index urls from user-provided options."""
  52. index_urls = []
  53. if not getattr(options, "no_index", False):
  54. url = getattr(options, "index_url", None)
  55. if url:
  56. index_urls.append(url)
  57. urls = getattr(options, "extra_index_urls", None)
  58. if urls:
  59. index_urls.extend(urls)
  60. # Return None rather than an empty list
  61. return index_urls or None
  62. def get_default_session(self, options: Values) -> PipSession:
  63. """Get a default-managed session."""
  64. if self._session is None:
  65. self._session = self.enter_context(self._build_session(options))
  66. # there's no type annotation on requests.Session, so it's
  67. # automatically ContextManager[Any] and self._session becomes Any,
  68. # then https://github.com/python/mypy/issues/7696 kicks in
  69. assert self._session is not None
  70. return self._session
  71. def _build_session(
  72. self,
  73. options: Values,
  74. retries: Optional[int] = None,
  75. timeout: Optional[int] = None,
  76. ) -> PipSession:
  77. assert not options.cache_dir or os.path.isabs(options.cache_dir)
  78. session = PipSession(
  79. cache=(
  80. os.path.join(options.cache_dir, "http") if options.cache_dir else None
  81. ),
  82. retries=retries if retries is not None else options.retries,
  83. trusted_hosts=options.trusted_hosts,
  84. index_urls=self._get_index_urls(options),
  85. )
  86. # Handle custom ca-bundles from the user
  87. if options.cert:
  88. session.verify = options.cert
  89. # Handle SSL client certificate
  90. if options.client_cert:
  91. session.cert = options.client_cert
  92. # Handle timeouts
  93. if options.timeout or timeout:
  94. session.timeout = timeout if timeout is not None else options.timeout
  95. # Handle configured proxies
  96. if options.proxy:
  97. session.proxies = {
  98. "http": options.proxy,
  99. "https": options.proxy,
  100. }
  101. # Determine if we can prompt the user for authentication or not
  102. session.auth.prompting = not options.no_input
  103. return session
  104. class IndexGroupCommand(Command, SessionCommandMixin):
  105. """
  106. Abstract base class for commands with the index_group options.
  107. This also corresponds to the commands that permit the pip version check.
  108. """
  109. def handle_pip_version_check(self, options: Values) -> None:
  110. """
  111. Do the pip version check if not disabled.
  112. This overrides the default behavior of not doing the check.
  113. """
  114. # Make sure the index_group options are present.
  115. assert hasattr(options, "no_index")
  116. if options.disable_pip_version_check or options.no_index:
  117. return
  118. # Otherwise, check if we're using the latest version of pip available.
  119. session = self._build_session(
  120. options, retries=0, timeout=min(5, options.timeout)
  121. )
  122. with session:
  123. pip_self_version_check(session, options)
  124. KEEPABLE_TEMPDIR_TYPES = [
  125. tempdir_kinds.BUILD_ENV,
  126. tempdir_kinds.EPHEM_WHEEL_CACHE,
  127. tempdir_kinds.REQ_BUILD,
  128. ]
  129. def warn_if_run_as_root() -> None:
  130. """Output a warning for sudo users on Unix.
  131. In a virtual environment, sudo pip still writes to virtualenv.
  132. On Windows, users may run pip as Administrator without issues.
  133. This warning only applies to Unix root users outside of virtualenv.
  134. """
  135. if running_under_virtualenv():
  136. return
  137. if not hasattr(os, "getuid"):
  138. return
  139. # On Windows, there are no "system managed" Python packages. Installing as
  140. # Administrator via pip is the correct way of updating system environments.
  141. #
  142. # We choose sys.platform over utils.compat.WINDOWS here to enable Mypy platform
  143. # checks: https://mypy.readthedocs.io/en/stable/common_issues.html
  144. if sys.platform == "win32" or sys.platform == "cygwin":
  145. return
  146. if os.getuid() != 0:
  147. return
  148. logger.warning(
  149. "Running pip as the 'root' user can result in broken permissions and "
  150. "conflicting behaviour with the system package manager. "
  151. "It is recommended to use a virtual environment instead: "
  152. "https://pip.pypa.io/warnings/venv"
  153. )
  154. def with_cleanup(func: Any) -> Any:
  155. """Decorator for common logic related to managing temporary
  156. directories.
  157. """
  158. def configure_tempdir_registry(registry: TempDirectoryTypeRegistry) -> None:
  159. for t in KEEPABLE_TEMPDIR_TYPES:
  160. registry.set_delete(t, False)
  161. def wrapper(
  162. self: RequirementCommand, options: Values, args: List[Any]
  163. ) -> Optional[int]:
  164. assert self.tempdir_registry is not None
  165. if options.no_clean:
  166. configure_tempdir_registry(self.tempdir_registry)
  167. try:
  168. return func(self, options, args)
  169. except PreviousBuildDirError:
  170. # This kind of conflict can occur when the user passes an explicit
  171. # build directory with a pre-existing folder. In that case we do
  172. # not want to accidentally remove it.
  173. configure_tempdir_registry(self.tempdir_registry)
  174. raise
  175. return wrapper
  176. class RequirementCommand(IndexGroupCommand):
  177. def __init__(self, *args: Any, **kw: Any) -> None:
  178. super().__init__(*args, **kw)
  179. self.cmd_opts.add_option(cmdoptions.no_clean())
  180. @staticmethod
  181. def determine_resolver_variant(options: Values) -> str:
  182. """Determines which resolver should be used, based on the given options."""
  183. if "legacy-resolver" in options.deprecated_features_enabled:
  184. return "legacy"
  185. return "2020-resolver"
  186. @staticmethod
  187. def determine_build_failure_suppression(options: Values) -> bool:
  188. """Determines whether build failures should be suppressed and backtracked on."""
  189. if "backtrack-on-build-failures" not in options.deprecated_features_enabled:
  190. return False
  191. if "legacy-resolver" in options.deprecated_features_enabled:
  192. raise CommandError("Cannot backtrack with legacy resolver.")
  193. deprecated(
  194. reason=(
  195. "Backtracking on build failures can mask issues related to how "
  196. "a package generates metadata or builds a wheel. This flag will "
  197. "be removed in pip 22.2."
  198. ),
  199. gone_in=None,
  200. replacement=(
  201. "avoiding known-bad versions by explicitly telling pip to ignore them "
  202. "(either directly as requirements, or via a constraints file)"
  203. ),
  204. feature_flag=None,
  205. issue=10655,
  206. )
  207. return True
  208. @classmethod
  209. def make_requirement_preparer(
  210. cls,
  211. temp_build_dir: TempDirectory,
  212. options: Values,
  213. req_tracker: RequirementTracker,
  214. session: PipSession,
  215. finder: PackageFinder,
  216. use_user_site: bool,
  217. download_dir: Optional[str] = None,
  218. verbosity: int = 0,
  219. ) -> RequirementPreparer:
  220. """
  221. Create a RequirementPreparer instance for the given parameters.
  222. """
  223. temp_build_dir_path = temp_build_dir.path
  224. assert temp_build_dir_path is not None
  225. resolver_variant = cls.determine_resolver_variant(options)
  226. if resolver_variant == "2020-resolver":
  227. lazy_wheel = "fast-deps" in options.features_enabled
  228. if lazy_wheel:
  229. logger.warning(
  230. "pip is using lazily downloaded wheels using HTTP "
  231. "range requests to obtain dependency information. "
  232. "This experimental feature is enabled through "
  233. "--use-feature=fast-deps and it is not ready for "
  234. "production."
  235. )
  236. else:
  237. lazy_wheel = False
  238. if "fast-deps" in options.features_enabled:
  239. logger.warning(
  240. "fast-deps has no effect when used with the legacy resolver."
  241. )
  242. in_tree_build = "out-of-tree-build" not in options.deprecated_features_enabled
  243. if "in-tree-build" in options.features_enabled:
  244. deprecated(
  245. reason="In-tree builds are now the default.",
  246. replacement="to remove the --use-feature=in-tree-build flag",
  247. gone_in="22.1",
  248. )
  249. if "out-of-tree-build" in options.deprecated_features_enabled:
  250. deprecated(
  251. reason="Out-of-tree builds are deprecated.",
  252. replacement=None,
  253. gone_in="22.1",
  254. )
  255. if options.progress_bar not in {"on", "off"}:
  256. deprecated(
  257. reason="Custom progress bar styles are deprecated",
  258. replacement="to use the default progress bar style.",
  259. gone_in="22.1",
  260. )
  261. return RequirementPreparer(
  262. build_dir=temp_build_dir_path,
  263. src_dir=options.src_dir,
  264. download_dir=download_dir,
  265. build_isolation=options.build_isolation,
  266. req_tracker=req_tracker,
  267. session=session,
  268. progress_bar=options.progress_bar,
  269. finder=finder,
  270. require_hashes=options.require_hashes,
  271. use_user_site=use_user_site,
  272. lazy_wheel=lazy_wheel,
  273. verbosity=verbosity,
  274. in_tree_build=in_tree_build,
  275. )
  276. @classmethod
  277. def make_resolver(
  278. cls,
  279. preparer: RequirementPreparer,
  280. finder: PackageFinder,
  281. options: Values,
  282. wheel_cache: Optional[WheelCache] = None,
  283. use_user_site: bool = False,
  284. ignore_installed: bool = True,
  285. ignore_requires_python: bool = False,
  286. force_reinstall: bool = False,
  287. upgrade_strategy: str = "to-satisfy-only",
  288. use_pep517: Optional[bool] = None,
  289. py_version_info: Optional[Tuple[int, ...]] = None,
  290. ) -> BaseResolver:
  291. """
  292. Create a Resolver instance for the given parameters.
  293. """
  294. make_install_req = partial(
  295. install_req_from_req_string,
  296. isolated=options.isolated_mode,
  297. use_pep517=use_pep517,
  298. )
  299. suppress_build_failures = cls.determine_build_failure_suppression(options)
  300. resolver_variant = cls.determine_resolver_variant(options)
  301. # The long import name and duplicated invocation is needed to convince
  302. # Mypy into correctly typechecking. Otherwise it would complain the
  303. # "Resolver" class being redefined.
  304. if resolver_variant == "2020-resolver":
  305. import pip._internal.resolution.resolvelib.resolver
  306. return pip._internal.resolution.resolvelib.resolver.Resolver(
  307. preparer=preparer,
  308. finder=finder,
  309. wheel_cache=wheel_cache,
  310. make_install_req=make_install_req,
  311. use_user_site=use_user_site,
  312. ignore_dependencies=options.ignore_dependencies,
  313. ignore_installed=ignore_installed,
  314. ignore_requires_python=ignore_requires_python,
  315. force_reinstall=force_reinstall,
  316. upgrade_strategy=upgrade_strategy,
  317. py_version_info=py_version_info,
  318. suppress_build_failures=suppress_build_failures,
  319. )
  320. import pip._internal.resolution.legacy.resolver
  321. return pip._internal.resolution.legacy.resolver.Resolver(
  322. preparer=preparer,
  323. finder=finder,
  324. wheel_cache=wheel_cache,
  325. make_install_req=make_install_req,
  326. use_user_site=use_user_site,
  327. ignore_dependencies=options.ignore_dependencies,
  328. ignore_installed=ignore_installed,
  329. ignore_requires_python=ignore_requires_python,
  330. force_reinstall=force_reinstall,
  331. upgrade_strategy=upgrade_strategy,
  332. py_version_info=py_version_info,
  333. )
  334. def get_requirements(
  335. self,
  336. args: List[str],
  337. options: Values,
  338. finder: PackageFinder,
  339. session: PipSession,
  340. ) -> List[InstallRequirement]:
  341. """
  342. Parse command-line arguments into the corresponding requirements.
  343. """
  344. requirements: List[InstallRequirement] = []
  345. for filename in options.constraints:
  346. for parsed_req in parse_requirements(
  347. filename,
  348. constraint=True,
  349. finder=finder,
  350. options=options,
  351. session=session,
  352. ):
  353. req_to_add = install_req_from_parsed_requirement(
  354. parsed_req,
  355. isolated=options.isolated_mode,
  356. user_supplied=False,
  357. )
  358. requirements.append(req_to_add)
  359. for req in args:
  360. req_to_add = install_req_from_line(
  361. req,
  362. None,
  363. isolated=options.isolated_mode,
  364. use_pep517=options.use_pep517,
  365. user_supplied=True,
  366. )
  367. requirements.append(req_to_add)
  368. for req in options.editables:
  369. req_to_add = install_req_from_editable(
  370. req,
  371. user_supplied=True,
  372. isolated=options.isolated_mode,
  373. use_pep517=options.use_pep517,
  374. )
  375. requirements.append(req_to_add)
  376. # NOTE: options.require_hashes may be set if --require-hashes is True
  377. for filename in options.requirements:
  378. for parsed_req in parse_requirements(
  379. filename, finder=finder, options=options, session=session
  380. ):
  381. req_to_add = install_req_from_parsed_requirement(
  382. parsed_req,
  383. isolated=options.isolated_mode,
  384. use_pep517=options.use_pep517,
  385. user_supplied=True,
  386. )
  387. requirements.append(req_to_add)
  388. # If any requirement has hash options, enable hash checking.
  389. if any(req.has_hash_options for req in requirements):
  390. options.require_hashes = True
  391. if not (args or options.editables or options.requirements):
  392. opts = {"name": self.name}
  393. if options.find_links:
  394. raise CommandError(
  395. "You must give at least one requirement to {name} "
  396. '(maybe you meant "pip {name} {links}"?)'.format(
  397. **dict(opts, links=" ".join(options.find_links))
  398. )
  399. )
  400. else:
  401. raise CommandError(
  402. "You must give at least one requirement to {name} "
  403. '(see "pip help {name}")'.format(**opts)
  404. )
  405. return requirements
  406. @staticmethod
  407. def trace_basic_info(finder: PackageFinder) -> None:
  408. """
  409. Trace basic information about the provided objects.
  410. """
  411. # Display where finder is looking for packages
  412. search_scope = finder.search_scope
  413. locations = search_scope.get_formatted_locations()
  414. if locations:
  415. logger.info(locations)
  416. def _build_package_finder(
  417. self,
  418. options: Values,
  419. session: PipSession,
  420. target_python: Optional[TargetPython] = None,
  421. ignore_requires_python: Optional[bool] = None,
  422. ) -> PackageFinder:
  423. """
  424. Create a package finder appropriate to this requirement command.
  425. :param ignore_requires_python: Whether to ignore incompatible
  426. "Requires-Python" values in links. Defaults to False.
  427. """
  428. link_collector = LinkCollector.create(session, options=options)
  429. selection_prefs = SelectionPreferences(
  430. allow_yanked=True,
  431. format_control=options.format_control,
  432. allow_all_prereleases=options.pre,
  433. prefer_binary=options.prefer_binary,
  434. ignore_requires_python=ignore_requires_python,
  435. )
  436. return PackageFinder.create(
  437. link_collector=link_collector,
  438. selection_prefs=selection_prefs,
  439. target_python=target_python,
  440. use_deprecated_html5lib="html5lib" in options.deprecated_features_enabled,
  441. )