expand_modules.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. import os
  2. import sys
  3. from typing import List, Pattern, Tuple
  4. from astroid import modutils
  5. from pylint.typing import ErrorDescriptionDict, ModuleDescriptionDict
  6. def _modpath_from_file(filename, is_namespace, path=None):
  7. def _is_package_cb(path, parts):
  8. return modutils.check_modpath_has_init(path, parts) or is_namespace
  9. return modutils.modpath_from_file_with_callback(
  10. filename, path=path, is_package_cb=_is_package_cb
  11. )
  12. def get_python_path(filepath: str) -> str:
  13. """TODO This get the python path with the (bad) assumption that there is always
  14. an __init__.py. This is not true since python 3.3 and is causing problem."""
  15. dirname = os.path.realpath(os.path.expanduser(filepath))
  16. if not os.path.isdir(dirname):
  17. dirname = os.path.dirname(dirname)
  18. while True:
  19. if not os.path.exists(os.path.join(dirname, "__init__.py")):
  20. return dirname
  21. old_dirname = dirname
  22. dirname = os.path.dirname(dirname)
  23. if old_dirname == dirname:
  24. return os.getcwd()
  25. def _is_in_ignore_list_re(element: str, ignore_list_re: List[Pattern]) -> bool:
  26. """determines if the element is matched in a regex ignore-list"""
  27. return any(file_pattern.match(element) for file_pattern in ignore_list_re)
  28. def expand_modules(
  29. files_or_modules: List[str],
  30. ignore_list: List[str],
  31. ignore_list_re: List[Pattern],
  32. ignore_list_paths_re: List[Pattern[str]],
  33. ) -> Tuple[List[ModuleDescriptionDict], List[ErrorDescriptionDict]]:
  34. """take a list of files/modules/packages and return the list of tuple
  35. (file, module name) which have to be actually checked
  36. """
  37. result: List[ModuleDescriptionDict] = []
  38. errors: List[ErrorDescriptionDict] = []
  39. path = sys.path.copy()
  40. for something in files_or_modules:
  41. basename = os.path.basename(something)
  42. if (
  43. basename in ignore_list
  44. or _is_in_ignore_list_re(os.path.basename(something), ignore_list_re)
  45. or _is_in_ignore_list_re(something, ignore_list_paths_re)
  46. ):
  47. continue
  48. module_path = get_python_path(something)
  49. additional_search_path = [".", module_path] + path
  50. if os.path.exists(something):
  51. # this is a file or a directory
  52. try:
  53. modname = ".".join(
  54. modutils.modpath_from_file(something, path=additional_search_path)
  55. )
  56. except ImportError:
  57. modname = os.path.splitext(basename)[0]
  58. if os.path.isdir(something):
  59. filepath = os.path.join(something, "__init__.py")
  60. else:
  61. filepath = something
  62. else:
  63. # suppose it's a module or package
  64. modname = something
  65. try:
  66. filepath = modutils.file_from_modpath(
  67. modname.split("."), path=additional_search_path
  68. )
  69. if filepath is None:
  70. continue
  71. except (ImportError, SyntaxError) as ex:
  72. # The SyntaxError is a Python bug and should be
  73. # removed once we move away from imp.find_module: https://bugs.python.org/issue10588
  74. errors.append({"key": "fatal", "mod": modname, "ex": ex})
  75. continue
  76. filepath = os.path.normpath(filepath)
  77. modparts = (modname or something).split(".")
  78. try:
  79. spec = modutils.file_info_from_modpath(
  80. modparts, path=additional_search_path
  81. )
  82. except ImportError:
  83. # Might not be acceptable, don't crash.
  84. is_namespace = False
  85. is_directory = os.path.isdir(something)
  86. else:
  87. is_namespace = modutils.is_namespace(spec)
  88. is_directory = modutils.is_directory(spec)
  89. if not is_namespace:
  90. result.append(
  91. {
  92. "path": filepath,
  93. "name": modname,
  94. "isarg": True,
  95. "basepath": filepath,
  96. "basename": modname,
  97. }
  98. )
  99. has_init = (
  100. not (modname.endswith(".__init__") or modname == "__init__")
  101. and os.path.basename(filepath) == "__init__.py"
  102. )
  103. if has_init or is_namespace or is_directory:
  104. for subfilepath in modutils.get_module_files(
  105. os.path.dirname(filepath), ignore_list, list_all=is_namespace
  106. ):
  107. if filepath == subfilepath:
  108. continue
  109. if _is_in_ignore_list_re(
  110. os.path.basename(subfilepath), ignore_list_re
  111. ) or _is_in_ignore_list_re(subfilepath, ignore_list_paths_re):
  112. continue
  113. modpath = _modpath_from_file(
  114. subfilepath, is_namespace, path=additional_search_path
  115. )
  116. submodname = ".".join(modpath)
  117. result.append(
  118. {
  119. "path": subfilepath,
  120. "name": submodname,
  121. "isarg": False,
  122. "basepath": filepath,
  123. "basename": modname,
  124. }
  125. )
  126. return result, errors