scripts.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2013-2015 Vinay Sajip.
  4. # Licensed to the Python Software Foundation under a contributor agreement.
  5. # See LICENSE.txt and CONTRIBUTORS.txt.
  6. #
  7. from io import BytesIO
  8. import logging
  9. import os
  10. import re
  11. import struct
  12. import sys
  13. from .compat import sysconfig, detect_encoding, ZipFile
  14. from .resources import finder
  15. from .util import (FileOperator, get_export_entry, convert_path,
  16. get_executable, in_venv)
  17. logger = logging.getLogger(__name__)
  18. _DEFAULT_MANIFEST = '''
  19. <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
  20. <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
  21. <assemblyIdentity version="1.0.0.0"
  22. processorArchitecture="X86"
  23. name="%s"
  24. type="win32"/>
  25. <!-- Identify the application security requirements. -->
  26. <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
  27. <security>
  28. <requestedPrivileges>
  29. <requestedExecutionLevel level="asInvoker" uiAccess="false"/>
  30. </requestedPrivileges>
  31. </security>
  32. </trustInfo>
  33. </assembly>'''.strip()
  34. # check if Python is called on the first line with this expression
  35. FIRST_LINE_RE = re.compile(b'^#!.*pythonw?[0-9.]*([ \t].*)?$')
  36. SCRIPT_TEMPLATE = r'''# -*- coding: utf-8 -*-
  37. import re
  38. import sys
  39. from %(module)s import %(import_name)s
  40. if __name__ == '__main__':
  41. sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
  42. sys.exit(%(func)s())
  43. '''
  44. def enquote_executable(executable):
  45. if ' ' in executable:
  46. # make sure we quote only the executable in case of env
  47. # for example /usr/bin/env "/dir with spaces/bin/jython"
  48. # instead of "/usr/bin/env /dir with spaces/bin/jython"
  49. # otherwise whole
  50. if executable.startswith('/usr/bin/env '):
  51. env, _executable = executable.split(' ', 1)
  52. if ' ' in _executable and not _executable.startswith('"'):
  53. executable = '%s "%s"' % (env, _executable)
  54. else:
  55. if not executable.startswith('"'):
  56. executable = '"%s"' % executable
  57. return executable
  58. # Keep the old name around (for now), as there is at least one project using it!
  59. _enquote_executable = enquote_executable
  60. class ScriptMaker(object):
  61. """
  62. A class to copy or create scripts from source scripts or callable
  63. specifications.
  64. """
  65. script_template = SCRIPT_TEMPLATE
  66. executable = None # for shebangs
  67. def __init__(self, source_dir, target_dir, add_launchers=True,
  68. dry_run=False, fileop=None):
  69. self.source_dir = source_dir
  70. self.target_dir = target_dir
  71. self.add_launchers = add_launchers
  72. self.force = False
  73. self.clobber = False
  74. # It only makes sense to set mode bits on POSIX.
  75. self.set_mode = (os.name == 'posix') or (os.name == 'java' and
  76. os._name == 'posix')
  77. self.variants = set(('', 'X.Y'))
  78. self._fileop = fileop or FileOperator(dry_run)
  79. self._is_nt = os.name == 'nt' or (
  80. os.name == 'java' and os._name == 'nt')
  81. self.version_info = sys.version_info
  82. def _get_alternate_executable(self, executable, options):
  83. if options.get('gui', False) and self._is_nt: # pragma: no cover
  84. dn, fn = os.path.split(executable)
  85. fn = fn.replace('python', 'pythonw')
  86. executable = os.path.join(dn, fn)
  87. return executable
  88. if sys.platform.startswith('java'): # pragma: no cover
  89. def _is_shell(self, executable):
  90. """
  91. Determine if the specified executable is a script
  92. (contains a #! line)
  93. """
  94. try:
  95. with open(executable) as fp:
  96. return fp.read(2) == '#!'
  97. except (OSError, IOError):
  98. logger.warning('Failed to open %s', executable)
  99. return False
  100. def _fix_jython_executable(self, executable):
  101. if self._is_shell(executable):
  102. # Workaround for Jython is not needed on Linux systems.
  103. import java
  104. if java.lang.System.getProperty('os.name') == 'Linux':
  105. return executable
  106. elif executable.lower().endswith('jython.exe'):
  107. # Use wrapper exe for Jython on Windows
  108. return executable
  109. return '/usr/bin/env %s' % executable
  110. def _build_shebang(self, executable, post_interp):
  111. """
  112. Build a shebang line. In the simple case (on Windows, or a shebang line
  113. which is not too long or contains spaces) use a simple formulation for
  114. the shebang. Otherwise, use /bin/sh as the executable, with a contrived
  115. shebang which allows the script to run either under Python or sh, using
  116. suitable quoting. Thanks to Harald Nordgren for his input.
  117. See also: http://www.in-ulm.de/~mascheck/various/shebang/#length
  118. https://hg.mozilla.org/mozilla-central/file/tip/mach
  119. """
  120. if os.name != 'posix':
  121. simple_shebang = True
  122. else:
  123. # Add 3 for '#!' prefix and newline suffix.
  124. shebang_length = len(executable) + len(post_interp) + 3
  125. if sys.platform == 'darwin':
  126. max_shebang_length = 512
  127. else:
  128. max_shebang_length = 127
  129. simple_shebang = ((b' ' not in executable) and
  130. (shebang_length <= max_shebang_length))
  131. if simple_shebang:
  132. result = b'#!' + executable + post_interp + b'\n'
  133. else:
  134. result = b'#!/bin/sh\n'
  135. result += b"'''exec' " + executable + post_interp + b' "$0" "$@"\n'
  136. result += b"' '''"
  137. return result
  138. def _get_shebang(self, encoding, post_interp=b'', options=None):
  139. enquote = True
  140. if self.executable:
  141. executable = self.executable
  142. enquote = False # assume this will be taken care of
  143. elif not sysconfig.is_python_build():
  144. executable = get_executable()
  145. elif in_venv(): # pragma: no cover
  146. executable = os.path.join(sysconfig.get_path('scripts'),
  147. 'python%s' % sysconfig.get_config_var('EXE'))
  148. else: # pragma: no cover
  149. executable = os.path.join(
  150. sysconfig.get_config_var('BINDIR'),
  151. 'python%s%s' % (sysconfig.get_config_var('VERSION'),
  152. sysconfig.get_config_var('EXE')))
  153. if options:
  154. executable = self._get_alternate_executable(executable, options)
  155. if sys.platform.startswith('java'): # pragma: no cover
  156. executable = self._fix_jython_executable(executable)
  157. # Normalise case for Windows - COMMENTED OUT
  158. # executable = os.path.normcase(executable)
  159. # N.B. The normalising operation above has been commented out: See
  160. # issue #124. Although paths in Windows are generally case-insensitive,
  161. # they aren't always. For example, a path containing a ẞ (which is a
  162. # LATIN CAPITAL LETTER SHARP S - U+1E9E) is normcased to ß (which is a
  163. # LATIN SMALL LETTER SHARP S' - U+00DF). The two are not considered by
  164. # Windows as equivalent in path names.
  165. # If the user didn't specify an executable, it may be necessary to
  166. # cater for executable paths with spaces (not uncommon on Windows)
  167. if enquote:
  168. executable = enquote_executable(executable)
  169. # Issue #51: don't use fsencode, since we later try to
  170. # check that the shebang is decodable using utf-8.
  171. executable = executable.encode('utf-8')
  172. # in case of IronPython, play safe and enable frames support
  173. if (sys.platform == 'cli' and '-X:Frames' not in post_interp
  174. and '-X:FullFrames' not in post_interp): # pragma: no cover
  175. post_interp += b' -X:Frames'
  176. shebang = self._build_shebang(executable, post_interp)
  177. # Python parser starts to read a script using UTF-8 until
  178. # it gets a #coding:xxx cookie. The shebang has to be the
  179. # first line of a file, the #coding:xxx cookie cannot be
  180. # written before. So the shebang has to be decodable from
  181. # UTF-8.
  182. try:
  183. shebang.decode('utf-8')
  184. except UnicodeDecodeError: # pragma: no cover
  185. raise ValueError(
  186. 'The shebang (%r) is not decodable from utf-8' % shebang)
  187. # If the script is encoded to a custom encoding (use a
  188. # #coding:xxx cookie), the shebang has to be decodable from
  189. # the script encoding too.
  190. if encoding != 'utf-8':
  191. try:
  192. shebang.decode(encoding)
  193. except UnicodeDecodeError: # pragma: no cover
  194. raise ValueError(
  195. 'The shebang (%r) is not decodable '
  196. 'from the script encoding (%r)' % (shebang, encoding))
  197. return shebang
  198. def _get_script_text(self, entry):
  199. return self.script_template % dict(module=entry.prefix,
  200. import_name=entry.suffix.split('.')[0],
  201. func=entry.suffix)
  202. manifest = _DEFAULT_MANIFEST
  203. def get_manifest(self, exename):
  204. base = os.path.basename(exename)
  205. return self.manifest % base
  206. def _write_script(self, names, shebang, script_bytes, filenames, ext):
  207. use_launcher = self.add_launchers and self._is_nt
  208. linesep = os.linesep.encode('utf-8')
  209. if not shebang.endswith(linesep):
  210. shebang += linesep
  211. if not use_launcher:
  212. script_bytes = shebang + script_bytes
  213. else: # pragma: no cover
  214. if ext == 'py':
  215. launcher = self._get_launcher('t')
  216. else:
  217. launcher = self._get_launcher('w')
  218. stream = BytesIO()
  219. with ZipFile(stream, 'w') as zf:
  220. zf.writestr('__main__.py', script_bytes)
  221. zip_data = stream.getvalue()
  222. script_bytes = launcher + shebang + zip_data
  223. for name in names:
  224. outname = os.path.join(self.target_dir, name)
  225. if use_launcher: # pragma: no cover
  226. n, e = os.path.splitext(outname)
  227. if e.startswith('.py'):
  228. outname = n
  229. outname = '%s.exe' % outname
  230. try:
  231. self._fileop.write_binary_file(outname, script_bytes)
  232. except Exception:
  233. # Failed writing an executable - it might be in use.
  234. logger.warning('Failed to write executable - trying to '
  235. 'use .deleteme logic')
  236. dfname = '%s.deleteme' % outname
  237. if os.path.exists(dfname):
  238. os.remove(dfname) # Not allowed to fail here
  239. os.rename(outname, dfname) # nor here
  240. self._fileop.write_binary_file(outname, script_bytes)
  241. logger.debug('Able to replace executable using '
  242. '.deleteme logic')
  243. try:
  244. os.remove(dfname)
  245. except Exception:
  246. pass # still in use - ignore error
  247. else:
  248. if self._is_nt and not outname.endswith('.' + ext): # pragma: no cover
  249. outname = '%s.%s' % (outname, ext)
  250. if os.path.exists(outname) and not self.clobber:
  251. logger.warning('Skipping existing file %s', outname)
  252. continue
  253. self._fileop.write_binary_file(outname, script_bytes)
  254. if self.set_mode:
  255. self._fileop.set_executable_mode([outname])
  256. filenames.append(outname)
  257. variant_separator = '-'
  258. def get_script_filenames(self, name):
  259. result = set()
  260. if '' in self.variants:
  261. result.add(name)
  262. if 'X' in self.variants:
  263. result.add('%s%s' % (name, self.version_info[0]))
  264. if 'X.Y' in self.variants:
  265. result.add('%s%s%s.%s' % (name, self.variant_separator,
  266. self.version_info[0], self.version_info[1]))
  267. return result
  268. def _make_script(self, entry, filenames, options=None):
  269. post_interp = b''
  270. if options:
  271. args = options.get('interpreter_args', [])
  272. if args:
  273. args = ' %s' % ' '.join(args)
  274. post_interp = args.encode('utf-8')
  275. shebang = self._get_shebang('utf-8', post_interp, options=options)
  276. script = self._get_script_text(entry).encode('utf-8')
  277. scriptnames = self.get_script_filenames(entry.name)
  278. if options and options.get('gui', False):
  279. ext = 'pyw'
  280. else:
  281. ext = 'py'
  282. self._write_script(scriptnames, shebang, script, filenames, ext)
  283. def _copy_script(self, script, filenames):
  284. adjust = False
  285. script = os.path.join(self.source_dir, convert_path(script))
  286. outname = os.path.join(self.target_dir, os.path.basename(script))
  287. if not self.force and not self._fileop.newer(script, outname):
  288. logger.debug('not copying %s (up-to-date)', script)
  289. return
  290. # Always open the file, but ignore failures in dry-run mode --
  291. # that way, we'll get accurate feedback if we can read the
  292. # script.
  293. try:
  294. f = open(script, 'rb')
  295. except IOError: # pragma: no cover
  296. if not self.dry_run:
  297. raise
  298. f = None
  299. else:
  300. first_line = f.readline()
  301. if not first_line: # pragma: no cover
  302. logger.warning('%s is an empty file (skipping)', script)
  303. return
  304. match = FIRST_LINE_RE.match(first_line.replace(b'\r\n', b'\n'))
  305. if match:
  306. adjust = True
  307. post_interp = match.group(1) or b''
  308. if not adjust:
  309. if f:
  310. f.close()
  311. self._fileop.copy_file(script, outname)
  312. if self.set_mode:
  313. self._fileop.set_executable_mode([outname])
  314. filenames.append(outname)
  315. else:
  316. logger.info('copying and adjusting %s -> %s', script,
  317. self.target_dir)
  318. if not self._fileop.dry_run:
  319. encoding, lines = detect_encoding(f.readline)
  320. f.seek(0)
  321. shebang = self._get_shebang(encoding, post_interp)
  322. if b'pythonw' in first_line: # pragma: no cover
  323. ext = 'pyw'
  324. else:
  325. ext = 'py'
  326. n = os.path.basename(outname)
  327. self._write_script([n], shebang, f.read(), filenames, ext)
  328. if f:
  329. f.close()
  330. @property
  331. def dry_run(self):
  332. return self._fileop.dry_run
  333. @dry_run.setter
  334. def dry_run(self, value):
  335. self._fileop.dry_run = value
  336. if os.name == 'nt' or (os.name == 'java' and os._name == 'nt'): # pragma: no cover
  337. # Executable launcher support.
  338. # Launchers are from https://bitbucket.org/vinay.sajip/simple_launcher/
  339. def _get_launcher(self, kind):
  340. if struct.calcsize('P') == 8: # 64-bit
  341. bits = '64'
  342. else:
  343. bits = '32'
  344. name = '%s%s.exe' % (kind, bits)
  345. # Issue 31: don't hardcode an absolute package name, but
  346. # determine it relative to the current package
  347. distlib_package = __name__.rsplit('.', 1)[0]
  348. resource = finder(distlib_package).find(name)
  349. if not resource:
  350. msg = ('Unable to find resource %s in package %s' % (name,
  351. distlib_package))
  352. raise ValueError(msg)
  353. return resource.bytes
  354. # Public API follows
  355. def make(self, specification, options=None):
  356. """
  357. Make a script.
  358. :param specification: The specification, which is either a valid export
  359. entry specification (to make a script from a
  360. callable) or a filename (to make a script by
  361. copying from a source location).
  362. :param options: A dictionary of options controlling script generation.
  363. :return: A list of all absolute pathnames written to.
  364. """
  365. filenames = []
  366. entry = get_export_entry(specification)
  367. if entry is None:
  368. self._copy_script(specification, filenames)
  369. else:
  370. self._make_script(entry, filenames, options=options)
  371. return filenames
  372. def make_multiple(self, specifications, options=None):
  373. """
  374. Take a list of specifications and make scripts from them,
  375. :param specifications: A list of specifications.
  376. :return: A list of all absolute pathnames written to,
  377. """
  378. filenames = []
  379. for specification in specifications:
  380. filenames.extend(self.make(specification, options))
  381. return filenames