wheel.py 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2013-2020 Vinay Sajip.
  4. # Licensed to the Python Software Foundation under a contributor agreement.
  5. # See LICENSE.txt and CONTRIBUTORS.txt.
  6. #
  7. from __future__ import unicode_literals
  8. import base64
  9. import codecs
  10. import datetime
  11. from email import message_from_file
  12. import hashlib
  13. import imp
  14. import json
  15. import logging
  16. import os
  17. import posixpath
  18. import re
  19. import shutil
  20. import sys
  21. import tempfile
  22. import zipfile
  23. from . import __version__, DistlibException
  24. from .compat import sysconfig, ZipFile, fsdecode, text_type, filter
  25. from .database import InstalledDistribution
  26. from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME,
  27. LEGACY_METADATA_FILENAME)
  28. from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache,
  29. cached_property, get_cache_base, read_exports, tempdir,
  30. get_platform)
  31. from .version import NormalizedVersion, UnsupportedVersionError
  32. logger = logging.getLogger(__name__)
  33. cache = None # created when needed
  34. if hasattr(sys, 'pypy_version_info'): # pragma: no cover
  35. IMP_PREFIX = 'pp'
  36. elif sys.platform.startswith('java'): # pragma: no cover
  37. IMP_PREFIX = 'jy'
  38. elif sys.platform == 'cli': # pragma: no cover
  39. IMP_PREFIX = 'ip'
  40. else:
  41. IMP_PREFIX = 'cp'
  42. VER_SUFFIX = sysconfig.get_config_var('py_version_nodot')
  43. if not VER_SUFFIX: # pragma: no cover
  44. if sys.version_info[1] >= 10:
  45. VER_SUFFIX = '%s_%s' % sys.version_info[:2] # PEP 641 (draft)
  46. else:
  47. VER_SUFFIX = '%s%s' % sys.version_info[:2]
  48. PYVER = 'py' + VER_SUFFIX
  49. IMPVER = IMP_PREFIX + VER_SUFFIX
  50. ARCH = get_platform().replace('-', '_').replace('.', '_')
  51. ABI = sysconfig.get_config_var('SOABI')
  52. if ABI and ABI.startswith('cpython-'):
  53. ABI = ABI.replace('cpython-', 'cp').split('-')[0]
  54. else:
  55. def _derive_abi():
  56. parts = ['cp', VER_SUFFIX]
  57. if sysconfig.get_config_var('Py_DEBUG'):
  58. parts.append('d')
  59. if sysconfig.get_config_var('WITH_PYMALLOC'):
  60. parts.append('m')
  61. if sysconfig.get_config_var('Py_UNICODE_SIZE') == 4:
  62. parts.append('u')
  63. return ''.join(parts)
  64. ABI = _derive_abi()
  65. del _derive_abi
  66. FILENAME_RE = re.compile(r'''
  67. (?P<nm>[^-]+)
  68. -(?P<vn>\d+[^-]*)
  69. (-(?P<bn>\d+[^-]*))?
  70. -(?P<py>\w+\d+(\.\w+\d+)*)
  71. -(?P<bi>\w+)
  72. -(?P<ar>\w+(\.\w+)*)
  73. \.whl$
  74. ''', re.IGNORECASE | re.VERBOSE)
  75. NAME_VERSION_RE = re.compile(r'''
  76. (?P<nm>[^-]+)
  77. -(?P<vn>\d+[^-]*)
  78. (-(?P<bn>\d+[^-]*))?$
  79. ''', re.IGNORECASE | re.VERBOSE)
  80. SHEBANG_RE = re.compile(br'\s*#![^\r\n]*')
  81. SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$')
  82. SHEBANG_PYTHON = b'#!python'
  83. SHEBANG_PYTHONW = b'#!pythonw'
  84. if os.sep == '/':
  85. to_posix = lambda o: o
  86. else:
  87. to_posix = lambda o: o.replace(os.sep, '/')
  88. class Mounter(object):
  89. def __init__(self):
  90. self.impure_wheels = {}
  91. self.libs = {}
  92. def add(self, pathname, extensions):
  93. self.impure_wheels[pathname] = extensions
  94. self.libs.update(extensions)
  95. def remove(self, pathname):
  96. extensions = self.impure_wheels.pop(pathname)
  97. for k, v in extensions:
  98. if k in self.libs:
  99. del self.libs[k]
  100. def find_module(self, fullname, path=None):
  101. if fullname in self.libs:
  102. result = self
  103. else:
  104. result = None
  105. return result
  106. def load_module(self, fullname):
  107. if fullname in sys.modules:
  108. result = sys.modules[fullname]
  109. else:
  110. if fullname not in self.libs:
  111. raise ImportError('unable to find extension for %s' % fullname)
  112. result = imp.load_dynamic(fullname, self.libs[fullname])
  113. result.__loader__ = self
  114. parts = fullname.rsplit('.', 1)
  115. if len(parts) > 1:
  116. result.__package__ = parts[0]
  117. return result
  118. _hook = Mounter()
  119. class Wheel(object):
  120. """
  121. Class to build and install from Wheel files (PEP 427).
  122. """
  123. wheel_version = (1, 1)
  124. hash_kind = 'sha256'
  125. def __init__(self, filename=None, sign=False, verify=False):
  126. """
  127. Initialise an instance using a (valid) filename.
  128. """
  129. self.sign = sign
  130. self.should_verify = verify
  131. self.buildver = ''
  132. self.pyver = [PYVER]
  133. self.abi = ['none']
  134. self.arch = ['any']
  135. self.dirname = os.getcwd()
  136. if filename is None:
  137. self.name = 'dummy'
  138. self.version = '0.1'
  139. self._filename = self.filename
  140. else:
  141. m = NAME_VERSION_RE.match(filename)
  142. if m:
  143. info = m.groupdict('')
  144. self.name = info['nm']
  145. # Reinstate the local version separator
  146. self.version = info['vn'].replace('_', '-')
  147. self.buildver = info['bn']
  148. self._filename = self.filename
  149. else:
  150. dirname, filename = os.path.split(filename)
  151. m = FILENAME_RE.match(filename)
  152. if not m:
  153. raise DistlibException('Invalid name or '
  154. 'filename: %r' % filename)
  155. if dirname:
  156. self.dirname = os.path.abspath(dirname)
  157. self._filename = filename
  158. info = m.groupdict('')
  159. self.name = info['nm']
  160. self.version = info['vn']
  161. self.buildver = info['bn']
  162. self.pyver = info['py'].split('.')
  163. self.abi = info['bi'].split('.')
  164. self.arch = info['ar'].split('.')
  165. @property
  166. def filename(self):
  167. """
  168. Build and return a filename from the various components.
  169. """
  170. if self.buildver:
  171. buildver = '-' + self.buildver
  172. else:
  173. buildver = ''
  174. pyver = '.'.join(self.pyver)
  175. abi = '.'.join(self.abi)
  176. arch = '.'.join(self.arch)
  177. # replace - with _ as a local version separator
  178. version = self.version.replace('-', '_')
  179. return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver,
  180. pyver, abi, arch)
  181. @property
  182. def exists(self):
  183. path = os.path.join(self.dirname, self.filename)
  184. return os.path.isfile(path)
  185. @property
  186. def tags(self):
  187. for pyver in self.pyver:
  188. for abi in self.abi:
  189. for arch in self.arch:
  190. yield pyver, abi, arch
  191. @cached_property
  192. def metadata(self):
  193. pathname = os.path.join(self.dirname, self.filename)
  194. name_ver = '%s-%s' % (self.name, self.version)
  195. info_dir = '%s.dist-info' % name_ver
  196. wrapper = codecs.getreader('utf-8')
  197. with ZipFile(pathname, 'r') as zf:
  198. wheel_metadata = self.get_wheel_metadata(zf)
  199. wv = wheel_metadata['Wheel-Version'].split('.', 1)
  200. file_version = tuple([int(i) for i in wv])
  201. # if file_version < (1, 1):
  202. # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME,
  203. # LEGACY_METADATA_FILENAME]
  204. # else:
  205. # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME]
  206. fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME]
  207. result = None
  208. for fn in fns:
  209. try:
  210. metadata_filename = posixpath.join(info_dir, fn)
  211. with zf.open(metadata_filename) as bf:
  212. wf = wrapper(bf)
  213. result = Metadata(fileobj=wf)
  214. if result:
  215. break
  216. except KeyError:
  217. pass
  218. if not result:
  219. raise ValueError('Invalid wheel, because metadata is '
  220. 'missing: looked in %s' % ', '.join(fns))
  221. return result
  222. def get_wheel_metadata(self, zf):
  223. name_ver = '%s-%s' % (self.name, self.version)
  224. info_dir = '%s.dist-info' % name_ver
  225. metadata_filename = posixpath.join(info_dir, 'WHEEL')
  226. with zf.open(metadata_filename) as bf:
  227. wf = codecs.getreader('utf-8')(bf)
  228. message = message_from_file(wf)
  229. return dict(message)
  230. @cached_property
  231. def info(self):
  232. pathname = os.path.join(self.dirname, self.filename)
  233. with ZipFile(pathname, 'r') as zf:
  234. result = self.get_wheel_metadata(zf)
  235. return result
  236. def process_shebang(self, data):
  237. m = SHEBANG_RE.match(data)
  238. if m:
  239. end = m.end()
  240. shebang, data_after_shebang = data[:end], data[end:]
  241. # Preserve any arguments after the interpreter
  242. if b'pythonw' in shebang.lower():
  243. shebang_python = SHEBANG_PYTHONW
  244. else:
  245. shebang_python = SHEBANG_PYTHON
  246. m = SHEBANG_DETAIL_RE.match(shebang)
  247. if m:
  248. args = b' ' + m.groups()[-1]
  249. else:
  250. args = b''
  251. shebang = shebang_python + args
  252. data = shebang + data_after_shebang
  253. else:
  254. cr = data.find(b'\r')
  255. lf = data.find(b'\n')
  256. if cr < 0 or cr > lf:
  257. term = b'\n'
  258. else:
  259. if data[cr:cr + 2] == b'\r\n':
  260. term = b'\r\n'
  261. else:
  262. term = b'\r'
  263. data = SHEBANG_PYTHON + term + data
  264. return data
  265. def get_hash(self, data, hash_kind=None):
  266. if hash_kind is None:
  267. hash_kind = self.hash_kind
  268. try:
  269. hasher = getattr(hashlib, hash_kind)
  270. except AttributeError:
  271. raise DistlibException('Unsupported hash algorithm: %r' % hash_kind)
  272. result = hasher(data).digest()
  273. result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii')
  274. return hash_kind, result
  275. def write_record(self, records, record_path, base):
  276. records = list(records) # make a copy, as mutated
  277. p = to_posix(os.path.relpath(record_path, base))
  278. records.append((p, '', ''))
  279. with CSVWriter(record_path) as writer:
  280. for row in records:
  281. writer.writerow(row)
  282. def write_records(self, info, libdir, archive_paths):
  283. records = []
  284. distinfo, info_dir = info
  285. hasher = getattr(hashlib, self.hash_kind)
  286. for ap, p in archive_paths:
  287. with open(p, 'rb') as f:
  288. data = f.read()
  289. digest = '%s=%s' % self.get_hash(data)
  290. size = os.path.getsize(p)
  291. records.append((ap, digest, size))
  292. p = os.path.join(distinfo, 'RECORD')
  293. self.write_record(records, p, libdir)
  294. ap = to_posix(os.path.join(info_dir, 'RECORD'))
  295. archive_paths.append((ap, p))
  296. def build_zip(self, pathname, archive_paths):
  297. with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:
  298. for ap, p in archive_paths:
  299. logger.debug('Wrote %s to %s in wheel', p, ap)
  300. zf.write(p, ap)
  301. def build(self, paths, tags=None, wheel_version=None):
  302. """
  303. Build a wheel from files in specified paths, and use any specified tags
  304. when determining the name of the wheel.
  305. """
  306. if tags is None:
  307. tags = {}
  308. libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]
  309. if libkey == 'platlib':
  310. is_pure = 'false'
  311. default_pyver = [IMPVER]
  312. default_abi = [ABI]
  313. default_arch = [ARCH]
  314. else:
  315. is_pure = 'true'
  316. default_pyver = [PYVER]
  317. default_abi = ['none']
  318. default_arch = ['any']
  319. self.pyver = tags.get('pyver', default_pyver)
  320. self.abi = tags.get('abi', default_abi)
  321. self.arch = tags.get('arch', default_arch)
  322. libdir = paths[libkey]
  323. name_ver = '%s-%s' % (self.name, self.version)
  324. data_dir = '%s.data' % name_ver
  325. info_dir = '%s.dist-info' % name_ver
  326. archive_paths = []
  327. # First, stuff which is not in site-packages
  328. for key in ('data', 'headers', 'scripts'):
  329. if key not in paths:
  330. continue
  331. path = paths[key]
  332. if os.path.isdir(path):
  333. for root, dirs, files in os.walk(path):
  334. for fn in files:
  335. p = fsdecode(os.path.join(root, fn))
  336. rp = os.path.relpath(p, path)
  337. ap = to_posix(os.path.join(data_dir, key, rp))
  338. archive_paths.append((ap, p))
  339. if key == 'scripts' and not p.endswith('.exe'):
  340. with open(p, 'rb') as f:
  341. data = f.read()
  342. data = self.process_shebang(data)
  343. with open(p, 'wb') as f:
  344. f.write(data)
  345. # Now, stuff which is in site-packages, other than the
  346. # distinfo stuff.
  347. path = libdir
  348. distinfo = None
  349. for root, dirs, files in os.walk(path):
  350. if root == path:
  351. # At the top level only, save distinfo for later
  352. # and skip it for now
  353. for i, dn in enumerate(dirs):
  354. dn = fsdecode(dn)
  355. if dn.endswith('.dist-info'):
  356. distinfo = os.path.join(root, dn)
  357. del dirs[i]
  358. break
  359. assert distinfo, '.dist-info directory expected, not found'
  360. for fn in files:
  361. # comment out next suite to leave .pyc files in
  362. if fsdecode(fn).endswith(('.pyc', '.pyo')):
  363. continue
  364. p = os.path.join(root, fn)
  365. rp = to_posix(os.path.relpath(p, path))
  366. archive_paths.append((rp, p))
  367. # Now distinfo. Assumed to be flat, i.e. os.listdir is enough.
  368. files = os.listdir(distinfo)
  369. for fn in files:
  370. if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):
  371. p = fsdecode(os.path.join(distinfo, fn))
  372. ap = to_posix(os.path.join(info_dir, fn))
  373. archive_paths.append((ap, p))
  374. wheel_metadata = [
  375. 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),
  376. 'Generator: distlib %s' % __version__,
  377. 'Root-Is-Purelib: %s' % is_pure,
  378. ]
  379. for pyver, abi, arch in self.tags:
  380. wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))
  381. p = os.path.join(distinfo, 'WHEEL')
  382. with open(p, 'w') as f:
  383. f.write('\n'.join(wheel_metadata))
  384. ap = to_posix(os.path.join(info_dir, 'WHEEL'))
  385. archive_paths.append((ap, p))
  386. # sort the entries by archive path. Not needed by any spec, but it
  387. # keeps the archive listing and RECORD tidier than they would otherwise
  388. # be. Use the number of path segments to keep directory entries together,
  389. # and keep the dist-info stuff at the end.
  390. def sorter(t):
  391. ap = t[0]
  392. n = ap.count('/')
  393. if '.dist-info' in ap:
  394. n += 10000
  395. return (n, ap)
  396. archive_paths = sorted(archive_paths, key=sorter)
  397. # Now, at last, RECORD.
  398. # Paths in here are archive paths - nothing else makes sense.
  399. self.write_records((distinfo, info_dir), libdir, archive_paths)
  400. # Now, ready to build the zip file
  401. pathname = os.path.join(self.dirname, self.filename)
  402. self.build_zip(pathname, archive_paths)
  403. return pathname
  404. def skip_entry(self, arcname):
  405. """
  406. Determine whether an archive entry should be skipped when verifying
  407. or installing.
  408. """
  409. # The signature file won't be in RECORD,
  410. # and we don't currently don't do anything with it
  411. # We also skip directories, as they won't be in RECORD
  412. # either. See:
  413. #
  414. # https://github.com/pypa/wheel/issues/294
  415. # https://github.com/pypa/wheel/issues/287
  416. # https://github.com/pypa/wheel/pull/289
  417. #
  418. return arcname.endswith(('/', '/RECORD.jws'))
  419. def install(self, paths, maker, **kwargs):
  420. """
  421. Install a wheel to the specified paths. If kwarg ``warner`` is
  422. specified, it should be a callable, which will be called with two
  423. tuples indicating the wheel version of this software and the wheel
  424. version in the file, if there is a discrepancy in the versions.
  425. This can be used to issue any warnings to raise any exceptions.
  426. If kwarg ``lib_only`` is True, only the purelib/platlib files are
  427. installed, and the headers, scripts, data and dist-info metadata are
  428. not written. If kwarg ``bytecode_hashed_invalidation`` is True, written
  429. bytecode will try to use file-hash based invalidation (PEP-552) on
  430. supported interpreter versions (CPython 2.7+).
  431. The return value is a :class:`InstalledDistribution` instance unless
  432. ``options.lib_only`` is True, in which case the return value is ``None``.
  433. """
  434. dry_run = maker.dry_run
  435. warner = kwargs.get('warner')
  436. lib_only = kwargs.get('lib_only', False)
  437. bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False)
  438. pathname = os.path.join(self.dirname, self.filename)
  439. name_ver = '%s-%s' % (self.name, self.version)
  440. data_dir = '%s.data' % name_ver
  441. info_dir = '%s.dist-info' % name_ver
  442. metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
  443. wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
  444. record_name = posixpath.join(info_dir, 'RECORD')
  445. wrapper = codecs.getreader('utf-8')
  446. with ZipFile(pathname, 'r') as zf:
  447. with zf.open(wheel_metadata_name) as bwf:
  448. wf = wrapper(bwf)
  449. message = message_from_file(wf)
  450. wv = message['Wheel-Version'].split('.', 1)
  451. file_version = tuple([int(i) for i in wv])
  452. if (file_version != self.wheel_version) and warner:
  453. warner(self.wheel_version, file_version)
  454. if message['Root-Is-Purelib'] == 'true':
  455. libdir = paths['purelib']
  456. else:
  457. libdir = paths['platlib']
  458. records = {}
  459. with zf.open(record_name) as bf:
  460. with CSVReader(stream=bf) as reader:
  461. for row in reader:
  462. p = row[0]
  463. records[p] = row
  464. data_pfx = posixpath.join(data_dir, '')
  465. info_pfx = posixpath.join(info_dir, '')
  466. script_pfx = posixpath.join(data_dir, 'scripts', '')
  467. # make a new instance rather than a copy of maker's,
  468. # as we mutate it
  469. fileop = FileOperator(dry_run=dry_run)
  470. fileop.record = True # so we can rollback if needed
  471. bc = not sys.dont_write_bytecode # Double negatives. Lovely!
  472. outfiles = [] # for RECORD writing
  473. # for script copying/shebang processing
  474. workdir = tempfile.mkdtemp()
  475. # set target dir later
  476. # we default add_launchers to False, as the
  477. # Python Launcher should be used instead
  478. maker.source_dir = workdir
  479. maker.target_dir = None
  480. try:
  481. for zinfo in zf.infolist():
  482. arcname = zinfo.filename
  483. if isinstance(arcname, text_type):
  484. u_arcname = arcname
  485. else:
  486. u_arcname = arcname.decode('utf-8')
  487. if self.skip_entry(u_arcname):
  488. continue
  489. row = records[u_arcname]
  490. if row[2] and str(zinfo.file_size) != row[2]:
  491. raise DistlibException('size mismatch for '
  492. '%s' % u_arcname)
  493. if row[1]:
  494. kind, value = row[1].split('=', 1)
  495. with zf.open(arcname) as bf:
  496. data = bf.read()
  497. _, digest = self.get_hash(data, kind)
  498. if digest != value:
  499. raise DistlibException('digest mismatch for '
  500. '%s' % arcname)
  501. if lib_only and u_arcname.startswith((info_pfx, data_pfx)):
  502. logger.debug('lib_only: skipping %s', u_arcname)
  503. continue
  504. is_script = (u_arcname.startswith(script_pfx)
  505. and not u_arcname.endswith('.exe'))
  506. if u_arcname.startswith(data_pfx):
  507. _, where, rp = u_arcname.split('/', 2)
  508. outfile = os.path.join(paths[where], convert_path(rp))
  509. else:
  510. # meant for site-packages.
  511. if u_arcname in (wheel_metadata_name, record_name):
  512. continue
  513. outfile = os.path.join(libdir, convert_path(u_arcname))
  514. if not is_script:
  515. with zf.open(arcname) as bf:
  516. fileop.copy_stream(bf, outfile)
  517. # Issue #147: permission bits aren't preserved. Using
  518. # zf.extract(zinfo, libdir) should have worked, but didn't,
  519. # see https://www.thetopsites.net/article/53834422.shtml
  520. # So ... manually preserve permission bits as given in zinfo
  521. if os.name == 'posix':
  522. # just set the normal permission bits
  523. os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF)
  524. outfiles.append(outfile)
  525. # Double check the digest of the written file
  526. if not dry_run and row[1]:
  527. with open(outfile, 'rb') as bf:
  528. data = bf.read()
  529. _, newdigest = self.get_hash(data, kind)
  530. if newdigest != digest:
  531. raise DistlibException('digest mismatch '
  532. 'on write for '
  533. '%s' % outfile)
  534. if bc and outfile.endswith('.py'):
  535. try:
  536. pyc = fileop.byte_compile(outfile,
  537. hashed_invalidation=bc_hashed_invalidation)
  538. outfiles.append(pyc)
  539. except Exception:
  540. # Don't give up if byte-compilation fails,
  541. # but log it and perhaps warn the user
  542. logger.warning('Byte-compilation failed',
  543. exc_info=True)
  544. else:
  545. fn = os.path.basename(convert_path(arcname))
  546. workname = os.path.join(workdir, fn)
  547. with zf.open(arcname) as bf:
  548. fileop.copy_stream(bf, workname)
  549. dn, fn = os.path.split(outfile)
  550. maker.target_dir = dn
  551. filenames = maker.make(fn)
  552. fileop.set_executable_mode(filenames)
  553. outfiles.extend(filenames)
  554. if lib_only:
  555. logger.debug('lib_only: returning None')
  556. dist = None
  557. else:
  558. # Generate scripts
  559. # Try to get pydist.json so we can see if there are
  560. # any commands to generate. If this fails (e.g. because
  561. # of a legacy wheel), log a warning but don't give up.
  562. commands = None
  563. file_version = self.info['Wheel-Version']
  564. if file_version == '1.0':
  565. # Use legacy info
  566. ep = posixpath.join(info_dir, 'entry_points.txt')
  567. try:
  568. with zf.open(ep) as bwf:
  569. epdata = read_exports(bwf)
  570. commands = {}
  571. for key in ('console', 'gui'):
  572. k = '%s_scripts' % key
  573. if k in epdata:
  574. commands['wrap_%s' % key] = d = {}
  575. for v in epdata[k].values():
  576. s = '%s:%s' % (v.prefix, v.suffix)
  577. if v.flags:
  578. s += ' [%s]' % ','.join(v.flags)
  579. d[v.name] = s
  580. except Exception:
  581. logger.warning('Unable to read legacy script '
  582. 'metadata, so cannot generate '
  583. 'scripts')
  584. else:
  585. try:
  586. with zf.open(metadata_name) as bwf:
  587. wf = wrapper(bwf)
  588. commands = json.load(wf).get('extensions')
  589. if commands:
  590. commands = commands.get('python.commands')
  591. except Exception:
  592. logger.warning('Unable to read JSON metadata, so '
  593. 'cannot generate scripts')
  594. if commands:
  595. console_scripts = commands.get('wrap_console', {})
  596. gui_scripts = commands.get('wrap_gui', {})
  597. if console_scripts or gui_scripts:
  598. script_dir = paths.get('scripts', '')
  599. if not os.path.isdir(script_dir):
  600. raise ValueError('Valid script path not '
  601. 'specified')
  602. maker.target_dir = script_dir
  603. for k, v in console_scripts.items():
  604. script = '%s = %s' % (k, v)
  605. filenames = maker.make(script)
  606. fileop.set_executable_mode(filenames)
  607. if gui_scripts:
  608. options = {'gui': True }
  609. for k, v in gui_scripts.items():
  610. script = '%s = %s' % (k, v)
  611. filenames = maker.make(script, options)
  612. fileop.set_executable_mode(filenames)
  613. p = os.path.join(libdir, info_dir)
  614. dist = InstalledDistribution(p)
  615. # Write SHARED
  616. paths = dict(paths) # don't change passed in dict
  617. del paths['purelib']
  618. del paths['platlib']
  619. paths['lib'] = libdir
  620. p = dist.write_shared_locations(paths, dry_run)
  621. if p:
  622. outfiles.append(p)
  623. # Write RECORD
  624. dist.write_installed_files(outfiles, paths['prefix'],
  625. dry_run)
  626. return dist
  627. except Exception: # pragma: no cover
  628. logger.exception('installation failed.')
  629. fileop.rollback()
  630. raise
  631. finally:
  632. shutil.rmtree(workdir)
  633. def _get_dylib_cache(self):
  634. global cache
  635. if cache is None:
  636. # Use native string to avoid issues on 2.x: see Python #20140.
  637. base = os.path.join(get_cache_base(), str('dylib-cache'),
  638. '%s.%s' % sys.version_info[:2])
  639. cache = Cache(base)
  640. return cache
  641. def _get_extensions(self):
  642. pathname = os.path.join(self.dirname, self.filename)
  643. name_ver = '%s-%s' % (self.name, self.version)
  644. info_dir = '%s.dist-info' % name_ver
  645. arcname = posixpath.join(info_dir, 'EXTENSIONS')
  646. wrapper = codecs.getreader('utf-8')
  647. result = []
  648. with ZipFile(pathname, 'r') as zf:
  649. try:
  650. with zf.open(arcname) as bf:
  651. wf = wrapper(bf)
  652. extensions = json.load(wf)
  653. cache = self._get_dylib_cache()
  654. prefix = cache.prefix_to_dir(pathname)
  655. cache_base = os.path.join(cache.base, prefix)
  656. if not os.path.isdir(cache_base):
  657. os.makedirs(cache_base)
  658. for name, relpath in extensions.items():
  659. dest = os.path.join(cache_base, convert_path(relpath))
  660. if not os.path.exists(dest):
  661. extract = True
  662. else:
  663. file_time = os.stat(dest).st_mtime
  664. file_time = datetime.datetime.fromtimestamp(file_time)
  665. info = zf.getinfo(relpath)
  666. wheel_time = datetime.datetime(*info.date_time)
  667. extract = wheel_time > file_time
  668. if extract:
  669. zf.extract(relpath, cache_base)
  670. result.append((name, dest))
  671. except KeyError:
  672. pass
  673. return result
  674. def is_compatible(self):
  675. """
  676. Determine if a wheel is compatible with the running system.
  677. """
  678. return is_compatible(self)
  679. def is_mountable(self):
  680. """
  681. Determine if a wheel is asserted as mountable by its metadata.
  682. """
  683. return True # for now - metadata details TBD
  684. def mount(self, append=False):
  685. pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
  686. if not self.is_compatible():
  687. msg = 'Wheel %s not compatible with this Python.' % pathname
  688. raise DistlibException(msg)
  689. if not self.is_mountable():
  690. msg = 'Wheel %s is marked as not mountable.' % pathname
  691. raise DistlibException(msg)
  692. if pathname in sys.path:
  693. logger.debug('%s already in path', pathname)
  694. else:
  695. if append:
  696. sys.path.append(pathname)
  697. else:
  698. sys.path.insert(0, pathname)
  699. extensions = self._get_extensions()
  700. if extensions:
  701. if _hook not in sys.meta_path:
  702. sys.meta_path.append(_hook)
  703. _hook.add(pathname, extensions)
  704. def unmount(self):
  705. pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
  706. if pathname not in sys.path:
  707. logger.debug('%s not in path', pathname)
  708. else:
  709. sys.path.remove(pathname)
  710. if pathname in _hook.impure_wheels:
  711. _hook.remove(pathname)
  712. if not _hook.impure_wheels:
  713. if _hook in sys.meta_path:
  714. sys.meta_path.remove(_hook)
  715. def verify(self):
  716. pathname = os.path.join(self.dirname, self.filename)
  717. name_ver = '%s-%s' % (self.name, self.version)
  718. data_dir = '%s.data' % name_ver
  719. info_dir = '%s.dist-info' % name_ver
  720. metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
  721. wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
  722. record_name = posixpath.join(info_dir, 'RECORD')
  723. wrapper = codecs.getreader('utf-8')
  724. with ZipFile(pathname, 'r') as zf:
  725. with zf.open(wheel_metadata_name) as bwf:
  726. wf = wrapper(bwf)
  727. message = message_from_file(wf)
  728. wv = message['Wheel-Version'].split('.', 1)
  729. file_version = tuple([int(i) for i in wv])
  730. # TODO version verification
  731. records = {}
  732. with zf.open(record_name) as bf:
  733. with CSVReader(stream=bf) as reader:
  734. for row in reader:
  735. p = row[0]
  736. records[p] = row
  737. for zinfo in zf.infolist():
  738. arcname = zinfo.filename
  739. if isinstance(arcname, text_type):
  740. u_arcname = arcname
  741. else:
  742. u_arcname = arcname.decode('utf-8')
  743. # See issue #115: some wheels have .. in their entries, but
  744. # in the filename ... e.g. __main__..py ! So the check is
  745. # updated to look for .. in the directory portions
  746. p = u_arcname.split('/')
  747. if '..' in p:
  748. raise DistlibException('invalid entry in '
  749. 'wheel: %r' % u_arcname)
  750. if self.skip_entry(u_arcname):
  751. continue
  752. row = records[u_arcname]
  753. if row[2] and str(zinfo.file_size) != row[2]:
  754. raise DistlibException('size mismatch for '
  755. '%s' % u_arcname)
  756. if row[1]:
  757. kind, value = row[1].split('=', 1)
  758. with zf.open(arcname) as bf:
  759. data = bf.read()
  760. _, digest = self.get_hash(data, kind)
  761. if digest != value:
  762. raise DistlibException('digest mismatch for '
  763. '%s' % arcname)
  764. def update(self, modifier, dest_dir=None, **kwargs):
  765. """
  766. Update the contents of a wheel in a generic way. The modifier should
  767. be a callable which expects a dictionary argument: its keys are
  768. archive-entry paths, and its values are absolute filesystem paths
  769. where the contents the corresponding archive entries can be found. The
  770. modifier is free to change the contents of the files pointed to, add
  771. new entries and remove entries, before returning. This method will
  772. extract the entire contents of the wheel to a temporary location, call
  773. the modifier, and then use the passed (and possibly updated)
  774. dictionary to write a new wheel. If ``dest_dir`` is specified, the new
  775. wheel is written there -- otherwise, the original wheel is overwritten.
  776. The modifier should return True if it updated the wheel, else False.
  777. This method returns the same value the modifier returns.
  778. """
  779. def get_version(path_map, info_dir):
  780. version = path = None
  781. key = '%s/%s' % (info_dir, LEGACY_METADATA_FILENAME)
  782. if key not in path_map:
  783. key = '%s/PKG-INFO' % info_dir
  784. if key in path_map:
  785. path = path_map[key]
  786. version = Metadata(path=path).version
  787. return version, path
  788. def update_version(version, path):
  789. updated = None
  790. try:
  791. v = NormalizedVersion(version)
  792. i = version.find('-')
  793. if i < 0:
  794. updated = '%s+1' % version
  795. else:
  796. parts = [int(s) for s in version[i + 1:].split('.')]
  797. parts[-1] += 1
  798. updated = '%s+%s' % (version[:i],
  799. '.'.join(str(i) for i in parts))
  800. except UnsupportedVersionError:
  801. logger.debug('Cannot update non-compliant (PEP-440) '
  802. 'version %r', version)
  803. if updated:
  804. md = Metadata(path=path)
  805. md.version = updated
  806. legacy = path.endswith(LEGACY_METADATA_FILENAME)
  807. md.write(path=path, legacy=legacy)
  808. logger.debug('Version updated from %r to %r', version,
  809. updated)
  810. pathname = os.path.join(self.dirname, self.filename)
  811. name_ver = '%s-%s' % (self.name, self.version)
  812. info_dir = '%s.dist-info' % name_ver
  813. record_name = posixpath.join(info_dir, 'RECORD')
  814. with tempdir() as workdir:
  815. with ZipFile(pathname, 'r') as zf:
  816. path_map = {}
  817. for zinfo in zf.infolist():
  818. arcname = zinfo.filename
  819. if isinstance(arcname, text_type):
  820. u_arcname = arcname
  821. else:
  822. u_arcname = arcname.decode('utf-8')
  823. if u_arcname == record_name:
  824. continue
  825. if '..' in u_arcname:
  826. raise DistlibException('invalid entry in '
  827. 'wheel: %r' % u_arcname)
  828. zf.extract(zinfo, workdir)
  829. path = os.path.join(workdir, convert_path(u_arcname))
  830. path_map[u_arcname] = path
  831. # Remember the version.
  832. original_version, _ = get_version(path_map, info_dir)
  833. # Files extracted. Call the modifier.
  834. modified = modifier(path_map, **kwargs)
  835. if modified:
  836. # Something changed - need to build a new wheel.
  837. current_version, path = get_version(path_map, info_dir)
  838. if current_version and (current_version == original_version):
  839. # Add or update local version to signify changes.
  840. update_version(current_version, path)
  841. # Decide where the new wheel goes.
  842. if dest_dir is None:
  843. fd, newpath = tempfile.mkstemp(suffix='.whl',
  844. prefix='wheel-update-',
  845. dir=workdir)
  846. os.close(fd)
  847. else:
  848. if not os.path.isdir(dest_dir):
  849. raise DistlibException('Not a directory: %r' % dest_dir)
  850. newpath = os.path.join(dest_dir, self.filename)
  851. archive_paths = list(path_map.items())
  852. distinfo = os.path.join(workdir, info_dir)
  853. info = distinfo, info_dir
  854. self.write_records(info, workdir, archive_paths)
  855. self.build_zip(newpath, archive_paths)
  856. if dest_dir is None:
  857. shutil.copyfile(newpath, pathname)
  858. return modified
  859. def _get_glibc_version():
  860. import platform
  861. ver = platform.libc_ver()
  862. result = []
  863. if ver[0] == 'glibc':
  864. for s in ver[1].split('.'):
  865. result.append(int(s) if s.isdigit() else 0)
  866. result = tuple(result)
  867. return result
  868. def compatible_tags():
  869. """
  870. Return (pyver, abi, arch) tuples compatible with this Python.
  871. """
  872. versions = [VER_SUFFIX]
  873. major = VER_SUFFIX[0]
  874. for minor in range(sys.version_info[1] - 1, - 1, -1):
  875. versions.append(''.join([major, str(minor)]))
  876. abis = []
  877. for suffix, _, _ in imp.get_suffixes():
  878. if suffix.startswith('.abi'):
  879. abis.append(suffix.split('.', 2)[1])
  880. abis.sort()
  881. if ABI != 'none':
  882. abis.insert(0, ABI)
  883. abis.append('none')
  884. result = []
  885. arches = [ARCH]
  886. if sys.platform == 'darwin':
  887. m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH)
  888. if m:
  889. name, major, minor, arch = m.groups()
  890. minor = int(minor)
  891. matches = [arch]
  892. if arch in ('i386', 'ppc'):
  893. matches.append('fat')
  894. if arch in ('i386', 'ppc', 'x86_64'):
  895. matches.append('fat3')
  896. if arch in ('ppc64', 'x86_64'):
  897. matches.append('fat64')
  898. if arch in ('i386', 'x86_64'):
  899. matches.append('intel')
  900. if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'):
  901. matches.append('universal')
  902. while minor >= 0:
  903. for match in matches:
  904. s = '%s_%s_%s_%s' % (name, major, minor, match)
  905. if s != ARCH: # already there
  906. arches.append(s)
  907. minor -= 1
  908. # Most specific - our Python version, ABI and arch
  909. for abi in abis:
  910. for arch in arches:
  911. result.append((''.join((IMP_PREFIX, versions[0])), abi, arch))
  912. # manylinux
  913. if abi != 'none' and sys.platform.startswith('linux'):
  914. arch = arch.replace('linux_', '')
  915. parts = _get_glibc_version()
  916. if len(parts) == 2:
  917. if parts >= (2, 5):
  918. result.append((''.join((IMP_PREFIX, versions[0])), abi,
  919. 'manylinux1_%s' % arch))
  920. if parts >= (2, 12):
  921. result.append((''.join((IMP_PREFIX, versions[0])), abi,
  922. 'manylinux2010_%s' % arch))
  923. if parts >= (2, 17):
  924. result.append((''.join((IMP_PREFIX, versions[0])), abi,
  925. 'manylinux2014_%s' % arch))
  926. result.append((''.join((IMP_PREFIX, versions[0])), abi,
  927. 'manylinux_%s_%s_%s' % (parts[0], parts[1],
  928. arch)))
  929. # where no ABI / arch dependency, but IMP_PREFIX dependency
  930. for i, version in enumerate(versions):
  931. result.append((''.join((IMP_PREFIX, version)), 'none', 'any'))
  932. if i == 0:
  933. result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any'))
  934. # no IMP_PREFIX, ABI or arch dependency
  935. for i, version in enumerate(versions):
  936. result.append((''.join(('py', version)), 'none', 'any'))
  937. if i == 0:
  938. result.append((''.join(('py', version[0])), 'none', 'any'))
  939. return set(result)
  940. COMPATIBLE_TAGS = compatible_tags()
  941. del compatible_tags
  942. def is_compatible(wheel, tags=None):
  943. if not isinstance(wheel, Wheel):
  944. wheel = Wheel(wheel) # assume it's a filename
  945. result = False
  946. if tags is None:
  947. tags = COMPATIBLE_TAGS
  948. for ver, abi, arch in tags:
  949. if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch:
  950. result = True
  951. break
  952. return result