imports.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023
  1. # Copyright (c) 2006-2015 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
  2. # Copyright (c) 2012-2014 Google, Inc.
  3. # Copyright (c) 2013 buck@yelp.com <buck@yelp.com>
  4. # Copyright (c) 2014-2020 Claudiu Popa <pcmanticore@gmail.com>
  5. # Copyright (c) 2014 Brett Cannon <brett@python.org>
  6. # Copyright (c) 2014 Arun Persaud <arun@nubati.net>
  7. # Copyright (c) 2015-2016 Moises Lopez <moylop260@vauxoo.com>
  8. # Copyright (c) 2015 Dmitry Pribysh <dmand@yandex.ru>
  9. # Copyright (c) 2015 Cezar <celnazli@bitdefender.com>
  10. # Copyright (c) 2015 Florian Bruhin <me@the-compiler.org>
  11. # Copyright (c) 2015 Noam Yorav-Raphael <noamraph@gmail.com>
  12. # Copyright (c) 2015 James Morgensen <james.morgensen@gmail.com>
  13. # Copyright (c) 2015 Ionel Cristian Maries <contact@ionelmc.ro>
  14. # Copyright (c) 2016, 2021 Ashley Whetter <ashley@awhetter.co.uk>
  15. # Copyright (c) 2016 Jared Garst <cultofjared@gmail.com>
  16. # Copyright (c) 2016 Maik Röder <maikroeder@gmail.com>
  17. # Copyright (c) 2016 Glenn Matthews <glenn@e-dad.net>
  18. # Copyright (c) 2017, 2020 hippo91 <guillaume.peillex@gmail.com>
  19. # Copyright (c) 2017 Michka Popoff <michkapopoff@gmail.com>
  20. # Copyright (c) 2017 Łukasz Rogalski <rogalski.91@gmail.com>
  21. # Copyright (c) 2017 Erik Wright <erik.wright@shopify.com>
  22. # Copyright (c) 2018 Lucas Cimon <lucas.cimon@gmail.com>
  23. # Copyright (c) 2018 Hornwitser <github@hornwitser.no>
  24. # Copyright (c) 2018 ssolanki <sushobhitsolanki@gmail.com>
  25. # Copyright (c) 2018 Natalie Serebryakova <natalie.serebryakova@Natalies-MacBook-Pro.local>
  26. # Copyright (c) 2018 Mike Frysinger <vapier@gmail.com>
  27. # Copyright (c) 2018 Sushobhit <31987769+sushobhit27@users.noreply.github.com>
  28. # Copyright (c) 2018 Marianna Polatoglou <mpolatoglou@bloomberg.net>
  29. # Copyright (c) 2019-2021 Pierre Sassoulas <pierre.sassoulas@gmail.com>
  30. # Copyright (c) 2019, 2021 Nick Drozd <nicholasdrozd@gmail.com>
  31. # Copyright (c) 2019 Hugo van Kemenade <hugovk@users.noreply.github.com>
  32. # Copyright (c) 2019 Nick Smith <clickthisnick@users.noreply.github.com>
  33. # Copyright (c) 2019 Paul Renvoisé <renvoisepaul@gmail.com>
  34. # Copyright (c) 2020 Peter Kolbus <peter.kolbus@gmail.com>
  35. # Copyright (c) 2020 Damien Baty <damien.baty@polyconseil.fr>
  36. # Copyright (c) 2020 Anthony Sottile <asottile@umich.edu>
  37. # Copyright (c) 2021 Tushar Sadhwani <tushar.sadhwani000@gmail.com>
  38. # Copyright (c) 2021 Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>
  39. # Copyright (c) 2021 Marc Mueller <30130371+cdce8p@users.noreply.github.com>
  40. # Copyright (c) 2021 Will Shanks <wsha@posteo.net>
  41. # Copyright (c) 2021 Matus Valo <matusvalo@users.noreply.github.com>
  42. # Copyright (c) 2021 Yu Shao, Pang <36848472+yushao2@users.noreply.github.com>
  43. # Copyright (c) 2021 Andrew Howe <howeaj@users.noreply.github.com>
  44. # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  45. # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
  46. """imports checkers for Python code"""
  47. import collections
  48. import copy
  49. import os
  50. import sys
  51. from distutils import sysconfig
  52. from typing import Any, Dict, List, Optional, Set, Tuple, Union
  53. import astroid
  54. from astroid import nodes
  55. from pylint.checkers import BaseChecker, DeprecatedMixin
  56. from pylint.checkers.utils import (
  57. check_messages,
  58. get_import_name,
  59. is_from_fallback_block,
  60. is_node_in_guarded_import_block,
  61. is_typing_guard,
  62. node_ignores_exception,
  63. )
  64. from pylint.exceptions import EmptyReportError
  65. from pylint.graph import DotBackend, get_cycles
  66. from pylint.interfaces import IAstroidChecker
  67. from pylint.lint import PyLinter
  68. from pylint.reporters.ureports.nodes import Paragraph, Section, VerbatimText
  69. from pylint.utils import IsortDriver, get_global_option
  70. def _qualified_names(modname):
  71. """Split the names of the given module into subparts
  72. For example,
  73. _qualified_names('pylint.checkers.ImportsChecker')
  74. returns
  75. ['pylint', 'pylint.checkers', 'pylint.checkers.ImportsChecker']
  76. """
  77. names = modname.split(".")
  78. return [".".join(names[0 : i + 1]) for i in range(len(names))]
  79. def _get_first_import(node, context, name, base, level, alias):
  80. """return the node where [base.]<name> is imported or None if not found"""
  81. fullname = f"{base}.{name}" if base else name
  82. first = None
  83. found = False
  84. for first in context.body:
  85. if first is node:
  86. continue
  87. if first.scope() is node.scope() and first.fromlineno > node.fromlineno:
  88. continue
  89. if isinstance(first, nodes.Import):
  90. if any(fullname == iname[0] for iname in first.names):
  91. found = True
  92. break
  93. elif isinstance(first, nodes.ImportFrom):
  94. if level == first.level:
  95. for imported_name, imported_alias in first.names:
  96. if fullname == f"{first.modname}.{imported_name}":
  97. found = True
  98. break
  99. if (
  100. name != "*"
  101. and name == imported_name
  102. and not (alias or imported_alias)
  103. ):
  104. found = True
  105. break
  106. if found:
  107. break
  108. if found and not astroid.are_exclusive(first, node):
  109. return first
  110. return None
  111. def _ignore_import_failure(node, modname, ignored_modules):
  112. for submodule in _qualified_names(modname):
  113. if submodule in ignored_modules:
  114. return True
  115. if is_node_in_guarded_import_block(node):
  116. # Ignore import failure if part of guarded import block
  117. # I.e. `sys.version_info` or `typing.TYPE_CHECKING`
  118. return True
  119. return node_ignores_exception(node, ImportError)
  120. # utilities to represents import dependencies as tree and dot graph ###########
  121. def _make_tree_defs(mod_files_list):
  122. """get a list of 2-uple (module, list_of_files_which_import_this_module),
  123. it will return a dictionary to represent this as a tree
  124. """
  125. tree_defs = {}
  126. for mod, files in mod_files_list:
  127. node = (tree_defs, ())
  128. for prefix in mod.split("."):
  129. node = node[0].setdefault(prefix, [{}, []])
  130. node[1] += files
  131. return tree_defs
  132. def _repr_tree_defs(data, indent_str=None):
  133. """return a string which represents imports as a tree"""
  134. lines = []
  135. nodes_items = data.items()
  136. for i, (mod, (sub, files)) in enumerate(sorted(nodes_items, key=lambda x: x[0])):
  137. files = "" if not files else f"({','.join(sorted(files))})"
  138. if indent_str is None:
  139. lines.append(f"{mod} {files}")
  140. sub_indent_str = " "
  141. else:
  142. lines.append(fr"{indent_str}\-{mod} {files}")
  143. if i == len(nodes_items) - 1:
  144. sub_indent_str = f"{indent_str} "
  145. else:
  146. sub_indent_str = f"{indent_str}| "
  147. if sub:
  148. lines.append(_repr_tree_defs(sub, sub_indent_str))
  149. return "\n".join(lines)
  150. def _dependencies_graph(filename: str, dep_info: Dict[str, Set[str]]) -> str:
  151. """write dependencies as a dot (graphviz) file"""
  152. done = {}
  153. printer = DotBackend(os.path.splitext(os.path.basename(filename))[0], rankdir="LR")
  154. printer.emit('URL="." node[shape="box"]')
  155. for modname, dependencies in sorted(dep_info.items()):
  156. sorted_dependencies = sorted(dependencies)
  157. done[modname] = 1
  158. printer.emit_node(modname)
  159. for depmodname in sorted_dependencies:
  160. if depmodname not in done:
  161. done[depmodname] = 1
  162. printer.emit_node(depmodname)
  163. for depmodname, dependencies in sorted(dep_info.items()):
  164. for modname in sorted(dependencies):
  165. printer.emit_edge(modname, depmodname)
  166. return printer.generate(filename)
  167. def _make_graph(
  168. filename: str, dep_info: Dict[str, Set[str]], sect: Section, gtype: str
  169. ):
  170. """generate a dependencies graph and add some information about it in the
  171. report's section
  172. """
  173. outputfile = _dependencies_graph(filename, dep_info)
  174. sect.append(Paragraph((f"{gtype}imports graph has been written to {outputfile}",)))
  175. # the import checker itself ###################################################
  176. MSGS = {
  177. "E0401": (
  178. "Unable to import %s",
  179. "import-error",
  180. "Used when pylint has been unable to import a module.",
  181. {"old_names": [("F0401", "old-import-error")]},
  182. ),
  183. "E0402": (
  184. "Attempted relative import beyond top-level package",
  185. "relative-beyond-top-level",
  186. "Used when a relative import tries to access too many levels "
  187. "in the current package.",
  188. ),
  189. "R0401": (
  190. "Cyclic import (%s)",
  191. "cyclic-import",
  192. "Used when a cyclic import between two or more modules is detected.",
  193. ),
  194. "R0402": (
  195. "Use 'from %s import %s' instead",
  196. "consider-using-from-import",
  197. "Emitted when a submodule of a package is imported and "
  198. "aliased with the same name. "
  199. "E.g., instead of ``import concurrent.futures as futures`` use "
  200. "``from concurrent import futures``",
  201. ),
  202. "W0401": (
  203. "Wildcard import %s",
  204. "wildcard-import",
  205. "Used when `from module import *` is detected.",
  206. ),
  207. "W0402": (
  208. "Uses of a deprecated module %r",
  209. "deprecated-module",
  210. "Used a module marked as deprecated is imported.",
  211. ),
  212. "W0404": (
  213. "Reimport %r (imported line %s)",
  214. "reimported",
  215. "Used when a module is reimported multiple times.",
  216. ),
  217. "W0406": (
  218. "Module import itself",
  219. "import-self",
  220. "Used when a module is importing itself.",
  221. ),
  222. "W0407": (
  223. "Prefer importing %r instead of %r",
  224. "preferred-module",
  225. "Used when a module imported has a preferred replacement module.",
  226. ),
  227. "W0410": (
  228. "__future__ import is not the first non docstring statement",
  229. "misplaced-future",
  230. "Python 2.5 and greater require __future__ import to be the "
  231. "first non docstring statement in the module.",
  232. ),
  233. "C0410": (
  234. "Multiple imports on one line (%s)",
  235. "multiple-imports",
  236. "Used when import statement importing multiple modules is detected.",
  237. ),
  238. "C0411": (
  239. "%s should be placed before %s",
  240. "wrong-import-order",
  241. "Used when PEP8 import order is not respected (standard imports "
  242. "first, then third-party libraries, then local imports)",
  243. ),
  244. "C0412": (
  245. "Imports from package %s are not grouped",
  246. "ungrouped-imports",
  247. "Used when imports are not grouped by packages",
  248. ),
  249. "C0413": (
  250. 'Import "%s" should be placed at the top of the module',
  251. "wrong-import-position",
  252. "Used when code and imports are mixed",
  253. ),
  254. "C0414": (
  255. "Import alias does not rename original package",
  256. "useless-import-alias",
  257. "Used when an import alias is same as original package."
  258. "e.g using import numpy as numpy instead of import numpy as np",
  259. ),
  260. "C0415": (
  261. "Import outside toplevel (%s)",
  262. "import-outside-toplevel",
  263. "Used when an import statement is used anywhere other than the module "
  264. "toplevel. Move this import to the top of the file.",
  265. ),
  266. }
  267. DEFAULT_STANDARD_LIBRARY = ()
  268. DEFAULT_KNOWN_THIRD_PARTY = ("enchant",)
  269. DEFAULT_PREFERRED_MODULES = ()
  270. class ImportsChecker(DeprecatedMixin, BaseChecker):
  271. """checks for
  272. * external modules dependencies
  273. * relative / wildcard imports
  274. * cyclic imports
  275. * uses of deprecated modules
  276. * uses of modules instead of preferred modules
  277. """
  278. __implements__ = IAstroidChecker
  279. name = "imports"
  280. msgs = MSGS
  281. priority = -2
  282. default_deprecated_modules = ()
  283. options = (
  284. (
  285. "deprecated-modules",
  286. {
  287. "default": default_deprecated_modules,
  288. "type": "csv",
  289. "metavar": "<modules>",
  290. "help": "Deprecated modules which should not be used,"
  291. " separated by a comma.",
  292. },
  293. ),
  294. (
  295. "preferred-modules",
  296. {
  297. "default": DEFAULT_PREFERRED_MODULES,
  298. "type": "csv",
  299. "metavar": "<module:preferred-module>",
  300. "help": "Couples of modules and preferred modules,"
  301. " separated by a comma.",
  302. },
  303. ),
  304. (
  305. "import-graph",
  306. {
  307. "default": "",
  308. "type": "string",
  309. "metavar": "<file.gv>",
  310. "help": "Output a graph (.gv or any supported image format) of"
  311. " all (i.e. internal and external) dependencies to the given file"
  312. " (report RP0402 must not be disabled).",
  313. },
  314. ),
  315. (
  316. "ext-import-graph",
  317. {
  318. "default": "",
  319. "type": "string",
  320. "metavar": "<file.gv>",
  321. "help": "Output a graph (.gv or any supported image format)"
  322. " of external dependencies to the given file"
  323. " (report RP0402 must not be disabled).",
  324. },
  325. ),
  326. (
  327. "int-import-graph",
  328. {
  329. "default": "",
  330. "type": "string",
  331. "metavar": "<file.gv>",
  332. "help": "Output a graph (.gv or any supported image format)"
  333. " of internal dependencies to the given file"
  334. " (report RP0402 must not be disabled).",
  335. },
  336. ),
  337. (
  338. "known-standard-library",
  339. {
  340. "default": DEFAULT_STANDARD_LIBRARY,
  341. "type": "csv",
  342. "metavar": "<modules>",
  343. "help": "Force import order to recognize a module as part of "
  344. "the standard compatibility libraries.",
  345. },
  346. ),
  347. (
  348. "known-third-party",
  349. {
  350. "default": DEFAULT_KNOWN_THIRD_PARTY,
  351. "type": "csv",
  352. "metavar": "<modules>",
  353. "help": "Force import order to recognize a module as part of "
  354. "a third party library.",
  355. },
  356. ),
  357. (
  358. "allow-any-import-level",
  359. {
  360. "default": (),
  361. "type": "csv",
  362. "metavar": "<modules>",
  363. "help": (
  364. "List of modules that can be imported at any level, not just "
  365. "the top level one."
  366. ),
  367. },
  368. ),
  369. (
  370. "analyse-fallback-blocks",
  371. {
  372. "default": False,
  373. "type": "yn",
  374. "metavar": "<y or n>",
  375. "help": "Analyse import fallback blocks. This can be used to "
  376. "support both Python 2 and 3 compatible code, which "
  377. "means that the block might have code that exists "
  378. "only in one or another interpreter, leading to false "
  379. "positives when analysed.",
  380. },
  381. ),
  382. (
  383. "allow-wildcard-with-all",
  384. {
  385. "default": False,
  386. "type": "yn",
  387. "metavar": "<y or n>",
  388. "help": "Allow wildcard imports from modules that define __all__.",
  389. },
  390. ),
  391. )
  392. def __init__(
  393. self, linter: Optional[PyLinter] = None
  394. ): # pylint: disable=super-init-not-called # See https://github.com/PyCQA/pylint/issues/4941
  395. BaseChecker.__init__(self, linter)
  396. self.import_graph: collections.defaultdict = collections.defaultdict(set)
  397. self._imports_stack: List[Tuple[Any, Any]] = []
  398. self._first_non_import_node = None
  399. self._module_pkg: Dict[
  400. Any, Any
  401. ] = {} # mapping of modules to the pkg they belong in
  402. self._allow_any_import_level: Set[Any] = set()
  403. self.reports = (
  404. ("RP0401", "External dependencies", self._report_external_dependencies),
  405. ("RP0402", "Modules dependencies graph", self._report_dependencies_graph),
  406. )
  407. self._site_packages = self._compute_site_packages()
  408. @staticmethod
  409. def _compute_site_packages():
  410. def _normalized_path(path):
  411. return os.path.normcase(os.path.abspath(path))
  412. paths = set()
  413. real_prefix = getattr(sys, "real_prefix", None)
  414. for prefix in filter(None, (real_prefix, sys.prefix)):
  415. path = sysconfig.get_python_lib(prefix=prefix)
  416. path = _normalized_path(path)
  417. paths.add(path)
  418. # Handle Debian's derivatives /usr/local.
  419. if os.path.isfile("/etc/debian_version"):
  420. for prefix in filter(None, (real_prefix, sys.prefix)):
  421. libpython = os.path.join(
  422. prefix,
  423. "local",
  424. "lib",
  425. "python" + sysconfig.get_python_version(),
  426. "dist-packages",
  427. )
  428. paths.add(libpython)
  429. return paths
  430. def open(self):
  431. """called before visiting project (i.e set of modules)"""
  432. self.linter.stats.dependencies = {}
  433. self.linter.stats = self.linter.stats
  434. self.import_graph = collections.defaultdict(set)
  435. self._module_pkg = {} # mapping of modules to the pkg they belong in
  436. self._excluded_edges = collections.defaultdict(set)
  437. self._ignored_modules = get_global_option(self, "ignored-modules", default=[])
  438. # Build a mapping {'module': 'preferred-module'}
  439. self.preferred_modules = dict(
  440. module.split(":")
  441. for module in self.config.preferred_modules
  442. if ":" in module
  443. )
  444. self._allow_any_import_level = set(self.config.allow_any_import_level)
  445. def _import_graph_without_ignored_edges(self):
  446. filtered_graph = copy.deepcopy(self.import_graph)
  447. for node in filtered_graph:
  448. filtered_graph[node].difference_update(self._excluded_edges[node])
  449. return filtered_graph
  450. def close(self):
  451. """called before visiting project (i.e set of modules)"""
  452. if self.linter.is_message_enabled("cyclic-import"):
  453. graph = self._import_graph_without_ignored_edges()
  454. vertices = list(graph)
  455. for cycle in get_cycles(graph, vertices=vertices):
  456. self.add_message("cyclic-import", args=" -> ".join(cycle))
  457. def deprecated_modules(self):
  458. """Callback returning the deprecated modules."""
  459. return self.config.deprecated_modules
  460. @check_messages(*MSGS)
  461. def visit_import(self, node: nodes.Import) -> None:
  462. """triggered when an import statement is seen"""
  463. self._check_reimport(node)
  464. self._check_import_as_rename(node)
  465. self._check_toplevel(node)
  466. names = [name for name, _ in node.names]
  467. if len(names) >= 2:
  468. self.add_message("multiple-imports", args=", ".join(names), node=node)
  469. for name in names:
  470. self.check_deprecated_module(node, name)
  471. self._check_preferred_module(node, name)
  472. imported_module = self._get_imported_module(node, name)
  473. if isinstance(node.parent, nodes.Module):
  474. # Allow imports nested
  475. self._check_position(node)
  476. if isinstance(node.scope(), nodes.Module):
  477. self._record_import(node, imported_module)
  478. if imported_module is None:
  479. continue
  480. self._add_imported_module(node, imported_module.name)
  481. @check_messages(*MSGS)
  482. def visit_importfrom(self, node: nodes.ImportFrom) -> None:
  483. """triggered when a from statement is seen"""
  484. basename = node.modname
  485. imported_module = self._get_imported_module(node, basename)
  486. self._check_import_as_rename(node)
  487. self._check_misplaced_future(node)
  488. self.check_deprecated_module(node, basename)
  489. self._check_preferred_module(node, basename)
  490. self._check_wildcard_imports(node, imported_module)
  491. self._check_same_line_imports(node)
  492. self._check_reimport(node, basename=basename, level=node.level)
  493. self._check_toplevel(node)
  494. if isinstance(node.parent, nodes.Module):
  495. # Allow imports nested
  496. self._check_position(node)
  497. if isinstance(node.scope(), nodes.Module):
  498. self._record_import(node, imported_module)
  499. if imported_module is None:
  500. return
  501. for name, _ in node.names:
  502. if name != "*":
  503. self._add_imported_module(node, f"{imported_module.name}.{name}")
  504. else:
  505. self._add_imported_module(node, imported_module.name)
  506. @check_messages(*MSGS)
  507. def leave_module(self, node: nodes.Module) -> None:
  508. # Check imports are grouped by category (standard, 3rd party, local)
  509. std_imports, ext_imports, loc_imports = self._check_imports_order(node)
  510. # Check that imports are grouped by package within a given category
  511. met_import: Set[str] = set() # set for 'import x' style
  512. met_from: Set[str] = set() # set for 'from x import y' style
  513. current_package = None
  514. for import_node, import_name in std_imports + ext_imports + loc_imports:
  515. if not self.linter.is_message_enabled(
  516. "ungrouped-imports", import_node.fromlineno
  517. ):
  518. continue
  519. met = met_from if isinstance(import_node, nodes.ImportFrom) else met_import
  520. package, _, _ = import_name.partition(".")
  521. if (
  522. current_package
  523. and current_package != package
  524. and package in met
  525. and is_node_in_guarded_import_block(import_node) is False
  526. ):
  527. self.add_message("ungrouped-imports", node=import_node, args=package)
  528. current_package = package
  529. met.add(package)
  530. self._imports_stack = []
  531. self._first_non_import_node = None
  532. def compute_first_non_import_node(self, node):
  533. if not self.linter.is_message_enabled("wrong-import-position", node.fromlineno):
  534. return
  535. # if the node does not contain an import instruction, and if it is the
  536. # first node of the module, keep a track of it (all the import positions
  537. # of the module will be compared to the position of this first
  538. # instruction)
  539. if self._first_non_import_node:
  540. return
  541. if not isinstance(node.parent, nodes.Module):
  542. return
  543. nested_allowed = [nodes.TryExcept, nodes.TryFinally]
  544. is_nested_allowed = [
  545. allowed for allowed in nested_allowed if isinstance(node, allowed)
  546. ]
  547. if is_nested_allowed and any(
  548. node.nodes_of_class((nodes.Import, nodes.ImportFrom))
  549. ):
  550. return
  551. if isinstance(node, nodes.Assign):
  552. # Add compatibility for module level dunder names
  553. # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
  554. valid_targets = [
  555. isinstance(target, nodes.AssignName)
  556. and target.name.startswith("__")
  557. and target.name.endswith("__")
  558. for target in node.targets
  559. ]
  560. if all(valid_targets):
  561. return
  562. self._first_non_import_node = node
  563. visit_tryfinally = (
  564. visit_tryexcept
  565. ) = (
  566. visit_assignattr
  567. ) = (
  568. visit_assign
  569. ) = (
  570. visit_ifexp
  571. ) = visit_comprehension = visit_expr = visit_if = compute_first_non_import_node
  572. def visit_functiondef(self, node: nodes.FunctionDef) -> None:
  573. if not self.linter.is_message_enabled("wrong-import-position", node.fromlineno):
  574. return
  575. # If it is the first non import instruction of the module, record it.
  576. if self._first_non_import_node:
  577. return
  578. # Check if the node belongs to an `If` or a `Try` block. If they
  579. # contain imports, skip recording this node.
  580. if not isinstance(node.parent.scope(), nodes.Module):
  581. return
  582. root = node
  583. while not isinstance(root.parent, nodes.Module):
  584. root = root.parent
  585. if isinstance(root, (nodes.If, nodes.TryFinally, nodes.TryExcept)):
  586. if any(root.nodes_of_class((nodes.Import, nodes.ImportFrom))):
  587. return
  588. self._first_non_import_node = node
  589. visit_classdef = visit_for = visit_while = visit_functiondef
  590. def _check_misplaced_future(self, node):
  591. basename = node.modname
  592. if basename == "__future__":
  593. # check if this is the first non-docstring statement in the module
  594. prev = node.previous_sibling()
  595. if prev:
  596. # consecutive future statements are possible
  597. if not (
  598. isinstance(prev, nodes.ImportFrom) and prev.modname == "__future__"
  599. ):
  600. self.add_message("misplaced-future", node=node)
  601. return
  602. def _check_same_line_imports(self, node):
  603. # Detect duplicate imports on the same line.
  604. names = (name for name, _ in node.names)
  605. counter = collections.Counter(names)
  606. for name, count in counter.items():
  607. if count > 1:
  608. self.add_message("reimported", node=node, args=(name, node.fromlineno))
  609. def _check_position(self, node):
  610. """Check `node` import or importfrom node position is correct
  611. Send a message if `node` comes before another instruction
  612. """
  613. # if a first non-import instruction has already been encountered,
  614. # it means the import comes after it and therefore is not well placed
  615. if self._first_non_import_node:
  616. self.add_message("wrong-import-position", node=node, args=node.as_string())
  617. def _record_import(self, node, importedmodnode):
  618. """Record the package `node` imports from"""
  619. if isinstance(node, nodes.ImportFrom):
  620. importedname = node.modname
  621. else:
  622. importedname = importedmodnode.name if importedmodnode else None
  623. if not importedname:
  624. importedname = node.names[0][0].split(".")[0]
  625. if isinstance(node, nodes.ImportFrom) and (node.level or 0) >= 1:
  626. # We need the importedname with first point to detect local package
  627. # Example of node:
  628. # 'from .my_package1 import MyClass1'
  629. # the output should be '.my_package1' instead of 'my_package1'
  630. # Example of node:
  631. # 'from . import my_package2'
  632. # the output should be '.my_package2' instead of '{pyfile}'
  633. importedname = "." + importedname
  634. self._imports_stack.append((node, importedname))
  635. @staticmethod
  636. def _is_fallback_import(node, imports):
  637. imports = [import_node for (import_node, _) in imports]
  638. return any(astroid.are_exclusive(import_node, node) for import_node in imports)
  639. def _check_imports_order(self, _module_node):
  640. """Checks imports of module `node` are grouped by category
  641. Imports must follow this order: standard, 3rd party, local
  642. """
  643. std_imports = []
  644. third_party_imports = []
  645. first_party_imports = []
  646. # need of a list that holds third or first party ordered import
  647. external_imports = []
  648. local_imports = []
  649. third_party_not_ignored = []
  650. first_party_not_ignored = []
  651. local_not_ignored = []
  652. isort_driver = IsortDriver(self.config)
  653. for node, modname in self._imports_stack:
  654. if modname.startswith("."):
  655. package = "." + modname.split(".")[1]
  656. else:
  657. package = modname.split(".")[0]
  658. nested = not isinstance(node.parent, nodes.Module)
  659. ignore_for_import_order = not self.linter.is_message_enabled(
  660. "wrong-import-order", node.fromlineno
  661. )
  662. import_category = isort_driver.place_module(package)
  663. node_and_package_import = (node, package)
  664. if import_category in {"FUTURE", "STDLIB"}:
  665. std_imports.append(node_and_package_import)
  666. wrong_import = (
  667. third_party_not_ignored
  668. or first_party_not_ignored
  669. or local_not_ignored
  670. )
  671. if self._is_fallback_import(node, wrong_import):
  672. continue
  673. if wrong_import and not nested:
  674. self.add_message(
  675. "wrong-import-order",
  676. node=node,
  677. args=(
  678. f'standard import "{node.as_string()}"',
  679. f'"{wrong_import[0][0].as_string()}"',
  680. ),
  681. )
  682. elif import_category == "THIRDPARTY":
  683. third_party_imports.append(node_and_package_import)
  684. external_imports.append(node_and_package_import)
  685. if not nested:
  686. if not ignore_for_import_order:
  687. third_party_not_ignored.append(node_and_package_import)
  688. else:
  689. self.linter.add_ignored_message(
  690. "wrong-import-order", node.fromlineno, node
  691. )
  692. wrong_import = first_party_not_ignored or local_not_ignored
  693. if wrong_import and not nested:
  694. self.add_message(
  695. "wrong-import-order",
  696. node=node,
  697. args=(
  698. f'third party import "{node.as_string()}"',
  699. f'"{wrong_import[0][0].as_string()}"',
  700. ),
  701. )
  702. elif import_category == "FIRSTPARTY":
  703. first_party_imports.append(node_and_package_import)
  704. external_imports.append(node_and_package_import)
  705. if not nested:
  706. if not ignore_for_import_order:
  707. first_party_not_ignored.append(node_and_package_import)
  708. else:
  709. self.linter.add_ignored_message(
  710. "wrong-import-order", node.fromlineno, node
  711. )
  712. wrong_import = local_not_ignored
  713. if wrong_import and not nested:
  714. self.add_message(
  715. "wrong-import-order",
  716. node=node,
  717. args=(
  718. f'first party import "{node.as_string()}"',
  719. f'"{wrong_import[0][0].as_string()}"',
  720. ),
  721. )
  722. elif import_category == "LOCALFOLDER":
  723. local_imports.append((node, package))
  724. if not nested:
  725. if not ignore_for_import_order:
  726. local_not_ignored.append((node, package))
  727. else:
  728. self.linter.add_ignored_message(
  729. "wrong-import-order", node.fromlineno, node
  730. )
  731. return std_imports, external_imports, local_imports
  732. def _get_imported_module(self, importnode, modname):
  733. try:
  734. return importnode.do_import_module(modname)
  735. except astroid.TooManyLevelsError:
  736. if _ignore_import_failure(importnode, modname, self._ignored_modules):
  737. return None
  738. self.add_message("relative-beyond-top-level", node=importnode)
  739. except astroid.AstroidSyntaxError as exc:
  740. message = f"Cannot import {modname!r} due to syntax error {str(exc.error)!r}" # pylint: disable=no-member; false positive
  741. self.add_message("syntax-error", line=importnode.lineno, args=message)
  742. except astroid.AstroidBuildingException:
  743. if not self.linter.is_message_enabled("import-error"):
  744. return None
  745. if _ignore_import_failure(importnode, modname, self._ignored_modules):
  746. return None
  747. if not self.config.analyse_fallback_blocks and is_from_fallback_block(
  748. importnode
  749. ):
  750. return None
  751. dotted_modname = get_import_name(importnode, modname)
  752. self.add_message("import-error", args=repr(dotted_modname), node=importnode)
  753. return None
  754. def _add_imported_module(
  755. self, node: Union[nodes.Import, nodes.ImportFrom], importedmodname: str
  756. ) -> None:
  757. """notify an imported module, used to analyze dependencies"""
  758. module_file = node.root().file
  759. context_name = node.root().name
  760. base = os.path.splitext(os.path.basename(module_file))[0]
  761. try:
  762. importedmodname = astroid.modutils.get_module_part(
  763. importedmodname, module_file
  764. )
  765. except ImportError:
  766. pass
  767. in_type_checking_block = isinstance(node.parent, nodes.If) and is_typing_guard(
  768. node.parent
  769. )
  770. if context_name == importedmodname:
  771. self.add_message("import-self", node=node)
  772. elif not astroid.modutils.is_standard_module(importedmodname):
  773. # if this is not a package __init__ module
  774. if base != "__init__" and context_name not in self._module_pkg:
  775. # record the module's parent, or the module itself if this is
  776. # a top level module, as the package it belongs to
  777. self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
  778. # handle dependencies
  779. dependencies_stat: Dict[str, Union[Set]] = self.linter.stats.dependencies
  780. importedmodnames = dependencies_stat.setdefault(importedmodname, set())
  781. if context_name not in importedmodnames:
  782. importedmodnames.add(context_name)
  783. # update import graph
  784. self.import_graph[context_name].add(importedmodname)
  785. if (
  786. not self.linter.is_message_enabled("cyclic-import", line=node.lineno)
  787. or in_type_checking_block
  788. ):
  789. self._excluded_edges[context_name].add(importedmodname)
  790. def _check_preferred_module(self, node, mod_path):
  791. """check if the module has a preferred replacement"""
  792. if mod_path in self.preferred_modules:
  793. self.add_message(
  794. "preferred-module",
  795. node=node,
  796. args=(self.preferred_modules[mod_path], mod_path),
  797. )
  798. def _check_import_as_rename(
  799. self, node: Union[nodes.Import, nodes.ImportFrom]
  800. ) -> None:
  801. names = node.names
  802. for name in names:
  803. if not all(name):
  804. return
  805. splitted_packages = name[0].rsplit(".", maxsplit=1)
  806. import_name = splitted_packages[-1]
  807. aliased_name = name[1]
  808. if import_name != aliased_name:
  809. continue
  810. if len(splitted_packages) == 1:
  811. self.add_message("useless-import-alias", node=node)
  812. elif len(splitted_packages) == 2:
  813. self.add_message(
  814. "consider-using-from-import",
  815. node=node,
  816. args=(splitted_packages[0], import_name),
  817. )
  818. def _check_reimport(self, node, basename=None, level=None):
  819. """check if the import is necessary (i.e. not already done)"""
  820. if not self.linter.is_message_enabled("reimported"):
  821. return
  822. frame = node.frame()
  823. root = node.root()
  824. contexts = [(frame, level)]
  825. if root is not frame:
  826. contexts.append((root, None))
  827. for known_context, known_level in contexts:
  828. for name, alias in node.names:
  829. first = _get_first_import(
  830. node, known_context, name, basename, known_level, alias
  831. )
  832. if first is not None:
  833. self.add_message(
  834. "reimported", node=node, args=(name, first.fromlineno)
  835. )
  836. def _report_external_dependencies(self, sect, _, _dummy):
  837. """return a verbatim layout for displaying dependencies"""
  838. dep_info = _make_tree_defs(self._external_dependencies_info().items())
  839. if not dep_info:
  840. raise EmptyReportError()
  841. tree_str = _repr_tree_defs(dep_info)
  842. sect.append(VerbatimText(tree_str))
  843. def _report_dependencies_graph(self, sect, _, _dummy):
  844. """write dependencies as a dot (graphviz) file"""
  845. dep_info = self.linter.stats.dependencies
  846. if not dep_info or not (
  847. self.config.import_graph
  848. or self.config.ext_import_graph
  849. or self.config.int_import_graph
  850. ):
  851. raise EmptyReportError()
  852. filename = self.config.import_graph
  853. if filename:
  854. _make_graph(filename, dep_info, sect, "")
  855. filename = self.config.ext_import_graph
  856. if filename:
  857. _make_graph(filename, self._external_dependencies_info(), sect, "external ")
  858. filename = self.config.int_import_graph
  859. if filename:
  860. _make_graph(filename, self._internal_dependencies_info(), sect, "internal ")
  861. def _filter_dependencies_graph(self, internal):
  862. """build the internal or the external dependency graph"""
  863. graph = collections.defaultdict(set)
  864. for importee, importers in self.linter.stats.dependencies.items():
  865. for importer in importers:
  866. package = self._module_pkg.get(importer, importer)
  867. is_inside = importee.startswith(package)
  868. if is_inside and internal or not is_inside and not internal:
  869. graph[importee].add(importer)
  870. return graph
  871. @astroid.decorators.cached
  872. def _external_dependencies_info(self):
  873. """return cached external dependencies information or build and
  874. cache them
  875. """
  876. return self._filter_dependencies_graph(internal=False)
  877. @astroid.decorators.cached
  878. def _internal_dependencies_info(self):
  879. """return cached internal dependencies information or build and
  880. cache them
  881. """
  882. return self._filter_dependencies_graph(internal=True)
  883. def _check_wildcard_imports(self, node, imported_module):
  884. if node.root().package:
  885. # Skip the check if in __init__.py issue #2026
  886. return
  887. wildcard_import_is_allowed = self._wildcard_import_is_allowed(imported_module)
  888. for name, _ in node.names:
  889. if name == "*" and not wildcard_import_is_allowed:
  890. self.add_message("wildcard-import", args=node.modname, node=node)
  891. def _wildcard_import_is_allowed(self, imported_module):
  892. return (
  893. self.config.allow_wildcard_with_all
  894. and imported_module is not None
  895. and "__all__" in imported_module.locals
  896. )
  897. def _check_toplevel(self, node):
  898. """Check whether the import is made outside the module toplevel."""
  899. # If the scope of the import is a module, then obviously it is
  900. # not outside the module toplevel.
  901. if isinstance(node.scope(), nodes.Module):
  902. return
  903. module_names = [
  904. f"{node.modname}.{name[0]}"
  905. if isinstance(node, nodes.ImportFrom)
  906. else name[0]
  907. for name in node.names
  908. ]
  909. # Get the full names of all the imports that are only allowed at the module level
  910. scoped_imports = [
  911. name for name in module_names if name not in self._allow_any_import_level
  912. ]
  913. if scoped_imports:
  914. self.add_message(
  915. "import-outside-toplevel", args=", ".join(scoped_imports), node=node
  916. )
  917. def register(linter):
  918. """required method to auto register this checker"""
  919. linter.register_checker(ImportsChecker(linter))