raw_building.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499
  1. # Copyright (c) 2006-2014 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
  2. # Copyright (c) 2012 FELD Boris <lothiraldan@gmail.com>
  3. # Copyright (c) 2014-2020 Claudiu Popa <pcmanticore@gmail.com>
  4. # Copyright (c) 2014 Google, Inc.
  5. # Copyright (c) 2015-2016 Ceridwen <ceridwenv@gmail.com>
  6. # Copyright (c) 2015 Florian Bruhin <me@the-compiler.org>
  7. # Copyright (c) 2015 Ovidiu Sabou <ovidiu@sabou.org>
  8. # Copyright (c) 2016 Derek Gustafson <degustaf@gmail.com>
  9. # Copyright (c) 2016 Jakub Wilk <jwilk@jwilk.net>
  10. # Copyright (c) 2018 Ville Skyttä <ville.skytta@iki.fi>
  11. # Copyright (c) 2018 Nick Drozd <nicholasdrozd@gmail.com>
  12. # Copyright (c) 2018 Bryce Guinta <bryce.paul.guinta@gmail.com>
  13. # Copyright (c) 2020-2021 hippo91 <guillaume.peillex@gmail.com>
  14. # Copyright (c) 2020 Becker Awqatty <bawqatty@mide.com>
  15. # Copyright (c) 2020 Robin Jarry <robin.jarry@6wind.com>
  16. # Copyright (c) 2021 Pierre Sassoulas <pierre.sassoulas@gmail.com>
  17. # Copyright (c) 2021 Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>
  18. # Copyright (c) 2021 Marc Mueller <30130371+cdce8p@users.noreply.github.com>
  19. # Copyright (c) 2021 Andrew Haigh <hello@nelf.in>
  20. # Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
  21. # For details: https://github.com/PyCQA/astroid/blob/main/LICENSE
  22. """this module contains a set of functions to create astroid trees from scratch
  23. (build_* functions) or from living object (object_build_* functions)
  24. """
  25. import builtins
  26. import inspect
  27. import os
  28. import sys
  29. import types
  30. import warnings
  31. from typing import List, Optional
  32. from astroid import bases, nodes
  33. from astroid.manager import AstroidManager
  34. from astroid.nodes import node_classes
  35. # the keys of CONST_CLS eg python builtin types
  36. _CONSTANTS = tuple(node_classes.CONST_CLS)
  37. _BUILTINS = vars(builtins)
  38. TYPE_NONE = type(None)
  39. TYPE_NOTIMPLEMENTED = type(NotImplemented)
  40. TYPE_ELLIPSIS = type(...)
  41. def _io_discrepancy(member):
  42. # _io module names itself `io`: http://bugs.python.org/issue18602
  43. member_self = getattr(member, "__self__", None)
  44. return (
  45. member_self
  46. and inspect.ismodule(member_self)
  47. and member_self.__name__ == "_io"
  48. and member.__module__ == "io"
  49. )
  50. def _attach_local_node(parent, node, name):
  51. node.name = name # needed by add_local_node
  52. parent.add_local_node(node)
  53. def _add_dunder_class(func, member):
  54. """Add a __class__ member to the given func node, if we can determine it."""
  55. python_cls = member.__class__
  56. cls_name = getattr(python_cls, "__name__", None)
  57. if not cls_name:
  58. return
  59. cls_bases = [ancestor.__name__ for ancestor in python_cls.__bases__]
  60. ast_klass = build_class(cls_name, cls_bases, python_cls.__doc__)
  61. func.instance_attrs["__class__"] = [ast_klass]
  62. _marker = object()
  63. def attach_dummy_node(node, name, runtime_object=_marker):
  64. """create a dummy node and register it in the locals of the given
  65. node with the specified name
  66. """
  67. enode = nodes.EmptyNode()
  68. enode.object = runtime_object
  69. _attach_local_node(node, enode, name)
  70. def _has_underlying_object(self):
  71. return self.object is not None and self.object is not _marker
  72. nodes.EmptyNode.has_underlying_object = _has_underlying_object
  73. def attach_const_node(node, name, value):
  74. """create a Const node and register it in the locals of the given
  75. node with the specified name
  76. """
  77. if name not in node.special_attributes:
  78. _attach_local_node(node, nodes.const_factory(value), name)
  79. def attach_import_node(node, modname, membername):
  80. """create a ImportFrom node and register it in the locals of the given
  81. node with the specified name
  82. """
  83. from_node = nodes.ImportFrom(modname, [(membername, None)])
  84. _attach_local_node(node, from_node, membername)
  85. def build_module(name: str, doc: Optional[str] = None) -> nodes.Module:
  86. """create and initialize an astroid Module node"""
  87. node = nodes.Module(name, doc, pure_python=False)
  88. node.package = False
  89. node.parent = None
  90. return node
  91. def build_class(name, basenames=(), doc=None):
  92. """create and initialize an astroid ClassDef node"""
  93. node = nodes.ClassDef(name, doc)
  94. for base in basenames:
  95. basenode = nodes.Name(name=base)
  96. node.bases.append(basenode)
  97. basenode.parent = node
  98. return node
  99. def build_function(
  100. name,
  101. args: Optional[List[str]] = None,
  102. posonlyargs: Optional[List[str]] = None,
  103. defaults=None,
  104. doc=None,
  105. kwonlyargs: Optional[List[str]] = None,
  106. ) -> nodes.FunctionDef:
  107. """create and initialize an astroid FunctionDef node"""
  108. # first argument is now a list of decorators
  109. func = nodes.FunctionDef(name, doc)
  110. func.args = argsnode = nodes.Arguments(parent=func)
  111. argsnode.postinit(
  112. args=[nodes.AssignName(name=arg, parent=argsnode) for arg in args or ()],
  113. defaults=[],
  114. kwonlyargs=[
  115. nodes.AssignName(name=arg, parent=argsnode) for arg in kwonlyargs or ()
  116. ],
  117. kw_defaults=[],
  118. annotations=[],
  119. posonlyargs=[
  120. nodes.AssignName(name=arg, parent=argsnode) for arg in posonlyargs or ()
  121. ],
  122. )
  123. for default in defaults or ():
  124. argsnode.defaults.append(nodes.const_factory(default))
  125. argsnode.defaults[-1].parent = argsnode
  126. if args:
  127. register_arguments(func)
  128. return func
  129. def build_from_import(fromname, names):
  130. """create and initialize an astroid ImportFrom import statement"""
  131. return nodes.ImportFrom(fromname, [(name, None) for name in names])
  132. def register_arguments(func, args=None):
  133. """add given arguments to local
  134. args is a list that may contains nested lists
  135. (i.e. def func(a, (b, c, d)): ...)
  136. """
  137. if args is None:
  138. args = func.args.args
  139. if func.args.vararg:
  140. func.set_local(func.args.vararg, func.args)
  141. if func.args.kwarg:
  142. func.set_local(func.args.kwarg, func.args)
  143. for arg in args:
  144. if isinstance(arg, nodes.AssignName):
  145. func.set_local(arg.name, arg)
  146. else:
  147. register_arguments(func, arg.elts)
  148. def object_build_class(node, member, localname):
  149. """create astroid for a living class object"""
  150. basenames = [base.__name__ for base in member.__bases__]
  151. return _base_class_object_build(node, member, basenames, localname=localname)
  152. def object_build_function(node, member, localname):
  153. """create astroid for a living function object"""
  154. signature = inspect.signature(member)
  155. args = []
  156. defaults = []
  157. posonlyargs = []
  158. kwonlyargs = []
  159. for param_name, param in signature.parameters.items():
  160. if param.kind == inspect.Parameter.POSITIONAL_ONLY:
  161. posonlyargs.append(param_name)
  162. elif param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD:
  163. args.append(param_name)
  164. elif param.kind == inspect.Parameter.VAR_POSITIONAL:
  165. args.append(param_name)
  166. elif param.kind == inspect.Parameter.VAR_KEYWORD:
  167. args.append(param_name)
  168. elif param.kind == inspect.Parameter.KEYWORD_ONLY:
  169. kwonlyargs.append(param_name)
  170. if param.default is not inspect._empty:
  171. defaults.append(param.default)
  172. func = build_function(
  173. getattr(member, "__name__", None) or localname,
  174. args,
  175. posonlyargs,
  176. defaults,
  177. member.__doc__,
  178. )
  179. node.add_local_node(func, localname)
  180. def object_build_datadescriptor(node, member, name):
  181. """create astroid for a living data descriptor object"""
  182. return _base_class_object_build(node, member, [], name)
  183. def object_build_methoddescriptor(node, member, localname):
  184. """create astroid for a living method descriptor object"""
  185. # FIXME get arguments ?
  186. func = build_function(
  187. getattr(member, "__name__", None) or localname, doc=member.__doc__
  188. )
  189. # set node's arguments to None to notice that we have no information, not
  190. # and empty argument list
  191. func.args.args = None
  192. node.add_local_node(func, localname)
  193. _add_dunder_class(func, member)
  194. def _base_class_object_build(node, member, basenames, name=None, localname=None):
  195. """create astroid for a living class object, with a given set of base names
  196. (e.g. ancestors)
  197. """
  198. klass = build_class(
  199. name or getattr(member, "__name__", None) or localname,
  200. basenames,
  201. member.__doc__,
  202. )
  203. klass._newstyle = isinstance(member, type)
  204. node.add_local_node(klass, localname)
  205. try:
  206. # limit the instantiation trick since it's too dangerous
  207. # (such as infinite test execution...)
  208. # this at least resolves common case such as Exception.args,
  209. # OSError.errno
  210. if issubclass(member, Exception):
  211. instdict = member().__dict__
  212. else:
  213. raise TypeError
  214. except TypeError:
  215. pass
  216. else:
  217. for item_name, obj in instdict.items():
  218. valnode = nodes.EmptyNode()
  219. valnode.object = obj
  220. valnode.parent = klass
  221. valnode.lineno = 1
  222. klass.instance_attrs[item_name] = [valnode]
  223. return klass
  224. def _build_from_function(node, name, member, module):
  225. # verify this is not an imported function
  226. try:
  227. code = member.__code__
  228. except AttributeError:
  229. # Some implementations don't provide the code object,
  230. # such as Jython.
  231. code = None
  232. filename = getattr(code, "co_filename", None)
  233. if filename is None:
  234. assert isinstance(member, object)
  235. object_build_methoddescriptor(node, member, name)
  236. elif filename != getattr(module, "__file__", None):
  237. attach_dummy_node(node, name, member)
  238. else:
  239. object_build_function(node, member, name)
  240. def _safe_has_attribute(obj, member):
  241. try:
  242. return hasattr(obj, member)
  243. except Exception: # pylint: disable=broad-except
  244. return False
  245. class InspectBuilder:
  246. """class for building nodes from living object
  247. this is actually a really minimal representation, including only Module,
  248. FunctionDef and ClassDef nodes and some others as guessed.
  249. """
  250. def __init__(self, manager_instance=None):
  251. self._manager = manager_instance or AstroidManager()
  252. self._done = {}
  253. self._module = None
  254. def inspect_build(
  255. self,
  256. module: types.ModuleType,
  257. modname: Optional[str] = None,
  258. path: Optional[str] = None,
  259. ) -> nodes.Module:
  260. """build astroid from a living module (i.e. using inspect)
  261. this is used when there is no python source code available (either
  262. because it's a built-in module or because the .py is not available)
  263. """
  264. self._module = module
  265. if modname is None:
  266. modname = module.__name__
  267. try:
  268. node = build_module(modname, module.__doc__)
  269. except AttributeError:
  270. # in jython, java modules have no __doc__ (see #109562)
  271. node = build_module(modname)
  272. node.file = node.path = os.path.abspath(path) if path else path
  273. node.name = modname
  274. self._manager.cache_module(node)
  275. node.package = hasattr(module, "__path__")
  276. self._done = {}
  277. self.object_build(node, module)
  278. return node
  279. def object_build(self, node, obj):
  280. """recursive method which create a partial ast from real objects
  281. (only function, class, and method are handled)
  282. """
  283. if obj in self._done:
  284. return self._done[obj]
  285. self._done[obj] = node
  286. for name in dir(obj):
  287. try:
  288. with warnings.catch_warnings():
  289. warnings.filterwarnings("error")
  290. member = getattr(obj, name)
  291. except (AttributeError, DeprecationWarning):
  292. # damned ExtensionClass.Base, I know you're there !
  293. attach_dummy_node(node, name)
  294. continue
  295. if inspect.ismethod(member):
  296. member = member.__func__
  297. if inspect.isfunction(member):
  298. _build_from_function(node, name, member, self._module)
  299. elif inspect.isbuiltin(member):
  300. if not _io_discrepancy(member) and self.imported_member(
  301. node, member, name
  302. ):
  303. continue
  304. object_build_methoddescriptor(node, member, name)
  305. elif inspect.isclass(member):
  306. if self.imported_member(node, member, name):
  307. continue
  308. if member in self._done:
  309. class_node = self._done[member]
  310. if class_node not in node.locals.get(name, ()):
  311. node.add_local_node(class_node, name)
  312. else:
  313. class_node = object_build_class(node, member, name)
  314. # recursion
  315. self.object_build(class_node, member)
  316. if name == "__class__" and class_node.parent is None:
  317. class_node.parent = self._done[self._module]
  318. elif inspect.ismethoddescriptor(member):
  319. assert isinstance(member, object)
  320. object_build_methoddescriptor(node, member, name)
  321. elif inspect.isdatadescriptor(member):
  322. assert isinstance(member, object)
  323. object_build_datadescriptor(node, member, name)
  324. elif isinstance(member, _CONSTANTS):
  325. attach_const_node(node, name, member)
  326. elif inspect.isroutine(member):
  327. # This should be called for Jython, where some builtin
  328. # methods aren't caught by isbuiltin branch.
  329. _build_from_function(node, name, member, self._module)
  330. elif _safe_has_attribute(member, "__all__"):
  331. module = build_module(name)
  332. _attach_local_node(node, module, name)
  333. # recursion
  334. self.object_build(module, member)
  335. else:
  336. # create an empty node so that the name is actually defined
  337. attach_dummy_node(node, name, member)
  338. return None
  339. def imported_member(self, node, member, name):
  340. """verify this is not an imported class or handle it"""
  341. # /!\ some classes like ExtensionClass doesn't have a __module__
  342. # attribute ! Also, this may trigger an exception on badly built module
  343. # (see http://www.logilab.org/ticket/57299 for instance)
  344. try:
  345. modname = getattr(member, "__module__", None)
  346. except TypeError:
  347. modname = None
  348. if modname is None:
  349. if name in {"__new__", "__subclasshook__"}:
  350. # Python 2.5.1 (r251:54863, Sep 1 2010, 22:03:14)
  351. # >>> print object.__new__.__module__
  352. # None
  353. modname = builtins.__name__
  354. else:
  355. attach_dummy_node(node, name, member)
  356. return True
  357. real_name = {"gtk": "gtk_gtk", "_io": "io"}.get(modname, modname)
  358. if real_name != self._module.__name__:
  359. # check if it sounds valid and then add an import node, else use a
  360. # dummy node
  361. try:
  362. getattr(sys.modules[modname], name)
  363. except (KeyError, AttributeError):
  364. attach_dummy_node(node, name, member)
  365. else:
  366. attach_import_node(node, modname, name)
  367. return True
  368. return False
  369. # astroid bootstrapping ######################################################
  370. _CONST_PROXY = {}
  371. def _set_proxied(const):
  372. # TODO : find a nicer way to handle this situation;
  373. return _CONST_PROXY[const.value.__class__]
  374. def _astroid_bootstrapping():
  375. """astroid bootstrapping the builtins module"""
  376. # this boot strapping is necessary since we need the Const nodes to
  377. # inspect_build builtins, and then we can proxy Const
  378. builder = InspectBuilder()
  379. astroid_builtin = builder.inspect_build(builtins)
  380. for cls, node_cls in node_classes.CONST_CLS.items():
  381. if cls is TYPE_NONE:
  382. proxy = build_class("NoneType")
  383. proxy.parent = astroid_builtin
  384. elif cls is TYPE_NOTIMPLEMENTED:
  385. proxy = build_class("NotImplementedType")
  386. proxy.parent = astroid_builtin
  387. elif cls is TYPE_ELLIPSIS:
  388. proxy = build_class("Ellipsis")
  389. proxy.parent = astroid_builtin
  390. else:
  391. proxy = astroid_builtin.getattr(cls.__name__)[0]
  392. if cls in (dict, list, set, tuple):
  393. node_cls._proxied = proxy
  394. else:
  395. _CONST_PROXY[cls] = proxy
  396. # Set the builtin module as parent for some builtins.
  397. nodes.Const._proxied = property(_set_proxied)
  398. _GeneratorType = nodes.ClassDef(
  399. types.GeneratorType.__name__, types.GeneratorType.__doc__
  400. )
  401. _GeneratorType.parent = astroid_builtin
  402. bases.Generator._proxied = _GeneratorType
  403. builder.object_build(bases.Generator._proxied, types.GeneratorType)
  404. if hasattr(types, "AsyncGeneratorType"):
  405. _AsyncGeneratorType = nodes.ClassDef(
  406. types.AsyncGeneratorType.__name__, types.AsyncGeneratorType.__doc__
  407. )
  408. _AsyncGeneratorType.parent = astroid_builtin
  409. bases.AsyncGenerator._proxied = _AsyncGeneratorType
  410. builder.object_build(bases.AsyncGenerator._proxied, types.AsyncGeneratorType)
  411. builtin_types = (
  412. types.GetSetDescriptorType,
  413. types.GeneratorType,
  414. types.MemberDescriptorType,
  415. TYPE_NONE,
  416. TYPE_NOTIMPLEMENTED,
  417. types.FunctionType,
  418. types.MethodType,
  419. types.BuiltinFunctionType,
  420. types.ModuleType,
  421. types.TracebackType,
  422. )
  423. for _type in builtin_types:
  424. if _type.__name__ not in astroid_builtin:
  425. cls = nodes.ClassDef(_type.__name__, _type.__doc__)
  426. cls.parent = astroid_builtin
  427. builder.object_build(cls, _type)
  428. astroid_builtin[_type.__name__] = cls
  429. _astroid_bootstrapping()