inspector.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. # Copyright (c) 2015-2020 Claudiu Popa <pcmanticore@gmail.com>
  2. # Copyright (c) 2017 Łukasz Rogalski <rogalski.91@gmail.com>
  3. # Copyright (c) 2018 ssolanki <sushobhitsolanki@gmail.com>
  4. # Copyright (c) 2018 Ville Skyttä <ville.skytta@iki.fi>
  5. # Copyright (c) 2019-2021 Pierre Sassoulas <pierre.sassoulas@gmail.com>
  6. # Copyright (c) 2019 Hugo van Kemenade <hugovk@users.noreply.github.com>
  7. # Copyright (c) 2020 hippo91 <guillaume.peillex@gmail.com>
  8. # Copyright (c) 2020 Anthony Sottile <asottile@umich.edu>
  9. # Copyright (c) 2021 Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>
  10. # Copyright (c) 2021 Marc Mueller <30130371+cdce8p@users.noreply.github.com>
  11. # Copyright (c) 2021 Nick Drozd <nicholasdrozd@gmail.com>
  12. # Copyright (c) 2021 Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com>
  13. # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  14. # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
  15. """
  16. Visitor doing some postprocessing on the astroid tree.
  17. Try to resolve definitions (namespace) dictionary, relationship...
  18. """
  19. import collections
  20. import os
  21. import traceback
  22. import astroid
  23. from astroid import nodes
  24. from pylint.pyreverse import utils
  25. def _iface_hdlr(_):
  26. """Handler used by interfaces to handle suspicious interface nodes."""
  27. return True
  28. def _astroid_wrapper(func, modname):
  29. print(f"parsing {modname}...")
  30. try:
  31. return func(modname)
  32. except astroid.exceptions.AstroidBuildingException as exc:
  33. print(exc)
  34. except Exception: # pylint: disable=broad-except
  35. traceback.print_exc()
  36. return None
  37. def interfaces(node, herited=True, handler_func=_iface_hdlr):
  38. """Return an iterator on interfaces implemented by the given class node."""
  39. try:
  40. implements = astroid.bases.Instance(node).getattr("__implements__")[0]
  41. except astroid.exceptions.NotFoundError:
  42. return
  43. if not herited and implements.frame() is not node:
  44. return
  45. found = set()
  46. missing = False
  47. for iface in nodes.unpack_infer(implements):
  48. if iface is astroid.Uninferable:
  49. missing = True
  50. continue
  51. if iface not in found and handler_func(iface):
  52. found.add(iface)
  53. yield iface
  54. if missing:
  55. raise astroid.exceptions.InferenceError()
  56. class IdGeneratorMixIn:
  57. """Mixin adding the ability to generate integer uid."""
  58. def __init__(self, start_value=0):
  59. self.id_count = start_value
  60. def init_counter(self, start_value=0):
  61. """init the id counter"""
  62. self.id_count = start_value
  63. def generate_id(self):
  64. """generate a new identifier"""
  65. self.id_count += 1
  66. return self.id_count
  67. class Project:
  68. """a project handle a set of modules / packages"""
  69. def __init__(self, name=""):
  70. self.name = name
  71. self.uid = None
  72. self.path = None
  73. self.modules = []
  74. self.locals = {}
  75. self.__getitem__ = self.locals.__getitem__
  76. self.__iter__ = self.locals.__iter__
  77. self.values = self.locals.values
  78. self.keys = self.locals.keys
  79. self.items = self.locals.items
  80. def add_module(self, node):
  81. self.locals[node.name] = node
  82. self.modules.append(node)
  83. def get_module(self, name):
  84. return self.locals[name]
  85. def get_children(self):
  86. return self.modules
  87. def __repr__(self):
  88. return f"<Project {self.name!r} at {id(self)} ({len(self.modules)} modules)>"
  89. class Linker(IdGeneratorMixIn, utils.LocalsVisitor):
  90. """Walk on the project tree and resolve relationships.
  91. According to options the following attributes may be
  92. added to visited nodes:
  93. * uid,
  94. a unique identifier for the node (on astroid.Project, astroid.Module,
  95. astroid.Class and astroid.locals_type). Only if the linker
  96. has been instantiated with tag=True parameter (False by default).
  97. * Function
  98. a mapping from locals names to their bounded value, which may be a
  99. constant like a string or an integer, or an astroid node
  100. (on astroid.Module, astroid.Class and astroid.Function).
  101. * instance_attrs_type
  102. as locals_type but for klass member attributes (only on astroid.Class)
  103. * implements,
  104. list of implemented interface _objects_ (only on astroid.Class nodes)
  105. """
  106. def __init__(self, project, inherited_interfaces=0, tag=False):
  107. IdGeneratorMixIn.__init__(self)
  108. utils.LocalsVisitor.__init__(self)
  109. # take inherited interface in consideration or not
  110. self.inherited_interfaces = inherited_interfaces
  111. # tag nodes or not
  112. self.tag = tag
  113. # visited project
  114. self.project = project
  115. def visit_project(self, node: Project) -> None:
  116. """visit a pyreverse.utils.Project node
  117. * optionally tag the node with a unique id
  118. """
  119. if self.tag:
  120. node.uid = self.generate_id()
  121. for module in node.modules:
  122. self.visit(module)
  123. def visit_module(self, node: nodes.Module) -> None:
  124. """visit an astroid.Module node
  125. * set the locals_type mapping
  126. * set the depends mapping
  127. * optionally tag the node with a unique id
  128. """
  129. if hasattr(node, "locals_type"):
  130. return
  131. node.locals_type = collections.defaultdict(list)
  132. node.depends = []
  133. if self.tag:
  134. node.uid = self.generate_id()
  135. def visit_classdef(self, node: nodes.ClassDef) -> None:
  136. """visit an astroid.Class node
  137. * set the locals_type and instance_attrs_type mappings
  138. * set the implements list and build it
  139. * optionally tag the node with a unique id
  140. """
  141. if hasattr(node, "locals_type"):
  142. return
  143. node.locals_type = collections.defaultdict(list)
  144. if self.tag:
  145. node.uid = self.generate_id()
  146. # resolve ancestors
  147. for baseobj in node.ancestors(recurs=False):
  148. specializations = getattr(baseobj, "specializations", [])
  149. specializations.append(node)
  150. baseobj.specializations = specializations
  151. # resolve instance attributes
  152. node.instance_attrs_type = collections.defaultdict(list)
  153. for assignattrs in node.instance_attrs.values():
  154. for assignattr in assignattrs:
  155. if not isinstance(assignattr, nodes.Unknown):
  156. self.handle_assignattr_type(assignattr, node)
  157. # resolve implemented interface
  158. try:
  159. node.implements = list(interfaces(node, self.inherited_interfaces))
  160. except astroid.InferenceError:
  161. node.implements = []
  162. def visit_functiondef(self, node: nodes.FunctionDef) -> None:
  163. """visit an astroid.Function node
  164. * set the locals_type mapping
  165. * optionally tag the node with a unique id
  166. """
  167. if hasattr(node, "locals_type"):
  168. return
  169. node.locals_type = collections.defaultdict(list)
  170. if self.tag:
  171. node.uid = self.generate_id()
  172. link_project = visit_project
  173. link_module = visit_module
  174. link_class = visit_classdef
  175. link_function = visit_functiondef
  176. def visit_assignname(self, node: nodes.AssignName) -> None:
  177. """visit an astroid.AssignName node
  178. handle locals_type
  179. """
  180. # avoid double parsing done by different Linkers.visit
  181. # running over the same project:
  182. if hasattr(node, "_handled"):
  183. return
  184. node._handled = True
  185. if node.name in node.frame():
  186. frame = node.frame()
  187. else:
  188. # the name has been defined as 'global' in the frame and belongs
  189. # there.
  190. frame = node.root()
  191. if not hasattr(frame, "locals_type"):
  192. # If the frame doesn't have a locals_type yet,
  193. # it means it wasn't yet visited. Visit it now
  194. # to add what's missing from it.
  195. if isinstance(frame, nodes.ClassDef):
  196. self.visit_classdef(frame)
  197. elif isinstance(frame, nodes.FunctionDef):
  198. self.visit_functiondef(frame)
  199. else:
  200. self.visit_module(frame)
  201. current = frame.locals_type[node.name]
  202. frame.locals_type[node.name] = list(set(current) | utils.infer_node(node))
  203. @staticmethod
  204. def handle_assignattr_type(node, parent):
  205. """handle an astroid.assignattr node
  206. handle instance_attrs_type
  207. """
  208. current = set(parent.instance_attrs_type[node.attrname])
  209. parent.instance_attrs_type[node.attrname] = list(
  210. current | utils.infer_node(node)
  211. )
  212. def visit_import(self, node: nodes.Import) -> None:
  213. """visit an astroid.Import node
  214. resolve module dependencies
  215. """
  216. context_file = node.root().file
  217. for name in node.names:
  218. relative = astroid.modutils.is_relative(name[0], context_file)
  219. self._imported_module(node, name[0], relative)
  220. def visit_importfrom(self, node: nodes.ImportFrom) -> None:
  221. """visit an astroid.ImportFrom node
  222. resolve module dependencies
  223. """
  224. basename = node.modname
  225. context_file = node.root().file
  226. if context_file is not None:
  227. relative = astroid.modutils.is_relative(basename, context_file)
  228. else:
  229. relative = False
  230. for name in node.names:
  231. if name[0] == "*":
  232. continue
  233. # analyze dependencies
  234. fullname = f"{basename}.{name[0]}"
  235. if fullname.find(".") > -1:
  236. try:
  237. fullname = astroid.modutils.get_module_part(fullname, context_file)
  238. except ImportError:
  239. continue
  240. if fullname != basename:
  241. self._imported_module(node, fullname, relative)
  242. def compute_module(self, context_name, mod_path):
  243. """return true if the module should be added to dependencies"""
  244. package_dir = os.path.dirname(self.project.path)
  245. if context_name == mod_path:
  246. return 0
  247. if astroid.modutils.is_standard_module(mod_path, (package_dir,)):
  248. return 1
  249. return 0
  250. def _imported_module(self, node, mod_path, relative):
  251. """Notify an imported module, used to analyze dependencies"""
  252. module = node.root()
  253. context_name = module.name
  254. if relative:
  255. mod_path = f"{'.'.join(context_name.split('.')[:-1])}.{mod_path}"
  256. if self.compute_module(context_name, mod_path):
  257. # handle dependencies
  258. if not hasattr(module, "depends"):
  259. module.depends = []
  260. mod_paths = module.depends
  261. if mod_path not in mod_paths:
  262. mod_paths.append(mod_path)
  263. def project_from_files(
  264. files, func_wrapper=_astroid_wrapper, project_name="no name", black_list=("CVS",)
  265. ):
  266. """return a Project from a list of files or modules"""
  267. # build the project representation
  268. astroid_manager = astroid.manager.AstroidManager()
  269. project = Project(project_name)
  270. for something in files:
  271. if not os.path.exists(something):
  272. fpath = astroid.modutils.file_from_modpath(something.split("."))
  273. elif os.path.isdir(something):
  274. fpath = os.path.join(something, "__init__.py")
  275. else:
  276. fpath = something
  277. ast = func_wrapper(astroid_manager.ast_from_file, fpath)
  278. if ast is None:
  279. continue
  280. project.path = project.path or ast.file
  281. project.add_module(ast)
  282. base_name = ast.name
  283. # recurse in package except if __init__ was explicitly given
  284. if ast.package and something.find("__init__") == -1:
  285. # recurse on others packages / modules if this is a package
  286. for fpath in astroid.modutils.get_module_files(
  287. os.path.dirname(ast.file), black_list
  288. ):
  289. ast = func_wrapper(astroid_manager.ast_from_file, fpath)
  290. if ast is None or ast.name == base_name:
  291. continue
  292. project.add_module(ast)
  293. return project