generate_release.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. # Generate a windows release and a generated embedded distribution of python
  2. # Latest python version is the argument of the script
  3. # Requirements: 7z, git
  4. # wine 32-bit is required in order to build on Linux
  5. import sys
  6. import urllib
  7. import urllib.request
  8. import subprocess
  9. import shutil
  10. import os
  11. import hashlib
  12. latest_version = sys.argv[1]
  13. def check(code):
  14. if code != 0:
  15. raise Exception('Got nonzero exit code from command')
  16. def check_subp(x):
  17. if x.returncode != 0:
  18. raise Exception('Got nonzero exit code from command')
  19. def log(line):
  20. print('[generate_release.py] ' + line)
  21. # https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions
  22. def remove_files_with_extensions(path, extensions):
  23. for root, dirs, files in os.walk(path):
  24. for file in files:
  25. if os.path.splitext(file)[1] in extensions:
  26. os.remove(os.path.join(root, file))
  27. def download_if_not_exists(file_name, url, sha256=None):
  28. if not os.path.exists('./' + file_name):
  29. log('Downloading ' + file_name + '..')
  30. data = urllib.request.urlopen(url).read()
  31. log('Finished downloading ' + file_name)
  32. with open('./' + file_name, 'wb') as f:
  33. f.write(data)
  34. if sha256:
  35. digest = hashlib.sha256(data).hexdigest()
  36. if digest != sha256:
  37. log('Error: ' + file_name + ' has wrong hash: ' + digest)
  38. sys.exit(1)
  39. else:
  40. log('Using existing ' + file_name)
  41. def wine_run_shell(command):
  42. if os.name == 'posix':
  43. check(os.system('wine ' + command.replace('\\', '/')))
  44. elif os.name == 'nt':
  45. check(os.system(command))
  46. else:
  47. raise Exception('Unsupported OS')
  48. def wine_run(command_parts):
  49. if os.name == 'posix':
  50. command_parts = ['wine', ] + command_parts
  51. if subprocess.run(command_parts).returncode != 0:
  52. raise Exception('Got nonzero exit code from command')
  53. # ---------- Get current release version, for later ----------
  54. log('Getting current release version')
  55. describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE)
  56. if describe_result.returncode != 0:
  57. raise Exception('Git describe failed')
  58. release_tag = describe_result.stdout.strip().decode('ascii')
  59. # ----------- Make copy of yt-local files using git -----------
  60. if os.path.exists('./yt-local'):
  61. log('Removing old release')
  62. shutil.rmtree('./yt-local')
  63. # Export git repository - this will ensure .git and things in gitignore won't
  64. # be included. Git only supports exporting archive formats, not into
  65. # directories, so pipe into 7z to put it into .\yt-local (not to be
  66. # confused with working directory. I'm calling it the same thing so it will
  67. # have that name when extracted from the final release zip archive)
  68. log('Making copy of yt-local files')
  69. check(os.system('git archive --format tar master | 7z x -si -ttar -oyt-local'))
  70. if len(os.listdir('./yt-local')) == 0:
  71. raise Exception('Failed to copy yt-local files')
  72. # ----------- Generate embedded python distribution -----------
  73. os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # *.pyc files double the size of the distribution
  74. get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
  75. latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version + '-embed-win32.zip'
  76. # I've verified that all the dlls in the following are signed by Microsoft.
  77. # Using this because Microsoft only provides installers whose files can't be
  78. # extracted without a special tool.
  79. visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x86.7z'
  80. visual_c_runtime_sha256 = '2549eb4d2ce4cf3a87425ea01940f74368bf1cda378ef8a8a1f1a12ed59f1547'
  81. download_if_not_exists('get-pip.py', get_pip_url)
  82. download_if_not_exists('python-dist-' + latest_version + '.zip', latest_dist_url)
  83. download_if_not_exists('vc15_(14.10.25017.0)_2017_x86.7z',
  84. visual_c_runtime_url,
  85. sha256=visual_c_runtime_sha256)
  86. if os.path.exists('./python'):
  87. log('Removing old python distribution')
  88. shutil.rmtree('./python')
  89. log('Extracting python distribution')
  90. check(os.system(r'7z -y x -opython python-dist-' + latest_version + '.zip'))
  91. log('Executing get-pip.py')
  92. wine_run(['./python/python.exe', '-I', 'get-pip.py'])
  93. '''
  94. # Explanation of .pth, ._pth, and isolated mode
  95. ## Isolated mode
  96. We want to run in what is called isolated mode, given by the switch -I.
  97. This mode prevents the embedded python distribution from searching in
  98. global directories for imports
  99. For example, if a user has `C:\Python37` and the embedded distribution is
  100. the same version, importing something using the embedded distribution will
  101. search `C:\Python37\Libs\site-packages`. This is not desirable because it
  102. means I might forget to distribute a dependency if I have it installed
  103. globally and I don't see any import errors. It also means that an outdated
  104. package might override the one being distributed and cause other problems.
  105. Isolated mode also means global environment variables and registry
  106. entries will be ignored
  107. ## The trouble with isolated mode
  108. Isolated mode also prevents the current working directory (cwd) from
  109. being added to `sys.path`. `sys.path` is the list of directories python will
  110. search in for imports. In non-isolated mode this is automatically populated
  111. with the cwd, `site-packages`, the directory of the python executable, etc.
  112. # How to get the cwd into sys.path in isolated mode
  113. The hack to get this to work is to use a .pth file. Normally, these files
  114. are just an additional list of directories to be added to `sys.path`.
  115. However, they also allow arbitrary code execution on lines beginning with
  116. `import ` (see https://docs.python.org/3/library/site.html). So, we simply
  117. add `import sys; sys.path.insert(0, '')` to add the cwd to path. `''` is
  118. shorthand for the cwd. See https://bugs.python.org/issue33698#msg318272
  119. # ._pth files in the embedded distribution
  120. A python37._pth file is included in the embedded distribution. The presence
  121. of tis file causes the embedded distribution to always use isolated mode
  122. (which we want). They are like .pth files, except they do not allow the
  123. arbitrary code execution trick. In my experimentation, I found that they
  124. prevent .pth files from loading. So the ._pth file will have to be removed
  125. and replaced with a .pth. Isolated mode will have to be specified manually.
  126. '''
  127. log('Removing ._pth')
  128. major_release = latest_version.split('.')[1]
  129. os.remove(r'./python/python3' + major_release + '._pth')
  130. log('Adding path_fixes.pth')
  131. with open(r'./python/path_fixes.pth', 'w', encoding='utf-8') as f:
  132. f.write("import sys; sys.path.insert(0, '')\n")
  133. '''# python3x._pth file tells the python executable where to look for files
  134. # Need to add the directory where packages are installed,
  135. # and the parent directory (which is where the yt-local files are)
  136. major_release = latest_version.split('.')[1]
  137. with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f:
  138. f.write('.\\Lib\\site-packages\n')
  139. f.write('..\n')'''
  140. log('Inserting Microsoft C Runtime')
  141. check_subp(subprocess.run([r'7z', '-y', 'e', '-opython', 'vc15_(14.10.25017.0)_2017_x86.7z', 'runtime_minimum/System']))
  142. log('Installing dependencies')
  143. wine_run(['./python/python.exe', '-I', '-m', 'pip', 'install', '--no-compile', '-r', './requirements.txt'])
  144. log('Uninstalling unnecessary gevent stuff')
  145. wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'cffi', 'pycparser'])
  146. shutil.rmtree(r'./python/Lib/site-packages/gevent/tests')
  147. shutil.rmtree(r'./python/Lib/site-packages/gevent/testing')
  148. remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation
  149. log('Uninstalling pip and others')
  150. wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'pip', 'wheel'])
  151. log('Removing pyc files') # Have to do this because get-pip and some packages don't respect --no-compile
  152. remove_files_with_extensions(r'./python', ['.pyc'])
  153. log('Removing dist-info and __pycache__')
  154. for root, dirs, files in os.walk(r'./python'):
  155. for dir in dirs:
  156. if dir == '__pycache__' or dir.endswith('.dist-info'):
  157. shutil.rmtree(os.path.join(root, dir))
  158. '''log('Removing get-pip.py and zipped distribution')
  159. os.remove(r'.\get-pip.py')
  160. os.remove(r'.\latest-dist.zip')'''
  161. print()
  162. log('Finished generating python distribution')
  163. # ----------- Copy generated distribution into release folder -----------
  164. log('Copying python distribution into release folder')
  165. shutil.copytree(r'./python', r'./yt-local/python')
  166. # ----------- Create release zip -----------
  167. output_filename = 'yt-local-' + release_tag + '-windows.zip'
  168. if os.path.exists('./' + output_filename):
  169. log('Removing previous zipped release')
  170. os.remove('./' + output_filename)
  171. log('Zipping release')
  172. check(os.system(r'7z -mx=9 a ' + output_filename + ' ./yt-local'))
  173. print('\n')
  174. log('Finished')