generate_release.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. # Generate a windows release and a generated embedded distribution of python
  2. # Latest python version is the argument of the script (or oldwin for
  3. # vista, 7 and 32-bit versions)
  4. # Requirements: 7z, git
  5. # wine is required in order to build on Linux
  6. import sys
  7. import urllib
  8. import urllib.request
  9. import subprocess
  10. import shutil
  11. import os
  12. import hashlib
  13. latest_version = sys.argv[1]
  14. if len(sys.argv) > 2:
  15. bitness = sys.argv[2]
  16. else:
  17. bitness = '64'
  18. if latest_version == 'oldwin':
  19. bitness = '32'
  20. latest_version = '3.7.9'
  21. suffix = 'windows-vista-7-only'
  22. else:
  23. suffix = 'windows'
  24. def check(code):
  25. if code != 0:
  26. raise Exception('Got nonzero exit code from command')
  27. def check_subp(x):
  28. if x.returncode != 0:
  29. raise Exception('Got nonzero exit code from command')
  30. def log(line):
  31. print('[generate_release.py] ' + line)
  32. # https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions
  33. def remove_files_with_extensions(path, extensions):
  34. for root, dirs, files in os.walk(path):
  35. for file in files:
  36. if os.path.splitext(file)[1] in extensions:
  37. os.remove(os.path.join(root, file))
  38. def download_if_not_exists(file_name, url, sha256=None):
  39. if not os.path.exists('./' + file_name):
  40. log('Downloading ' + file_name + '..')
  41. data = urllib.request.urlopen(url).read()
  42. log('Finished downloading ' + file_name)
  43. with open('./' + file_name, 'wb') as f:
  44. f.write(data)
  45. if sha256:
  46. digest = hashlib.sha256(data).hexdigest()
  47. if digest != sha256:
  48. log('Error: ' + file_name + ' has wrong hash: ' + digest)
  49. sys.exit(1)
  50. else:
  51. log('Using existing ' + file_name)
  52. def wine_run_shell(command):
  53. if os.name == 'posix':
  54. check(os.system('wine ' + command.replace('\\', '/')))
  55. elif os.name == 'nt':
  56. check(os.system(command))
  57. else:
  58. raise Exception('Unsupported OS')
  59. def wine_run(command_parts):
  60. if os.name == 'posix':
  61. command_parts = ['wine',] + command_parts
  62. if subprocess.run(command_parts).returncode != 0:
  63. raise Exception('Got nonzero exit code from command')
  64. # ---------- Get current release version, for later ----------
  65. log('Getting current release version')
  66. describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE)
  67. if describe_result.returncode != 0:
  68. raise Exception('Git describe failed')
  69. release_tag = describe_result.stdout.strip().decode('ascii')
  70. # ----------- Make copy of yt-local files using git -----------
  71. if os.path.exists('./yt-local'):
  72. log('Removing old release')
  73. shutil.rmtree('./yt-local')
  74. # Export git repository - this will ensure .git and things in gitignore won't
  75. # be included. Git only supports exporting archive formats, not into
  76. # directories, so pipe into 7z to put it into .\yt-local (not to be
  77. # confused with working directory. I'm calling it the same thing so it will
  78. # have that name when extracted from the final release zip archive)
  79. log('Making copy of yt-local files')
  80. check(os.system('git archive --format tar master | 7z x -si -ttar -oyt-local'))
  81. if len(os.listdir('./yt-local')) == 0:
  82. raise Exception('Failed to copy yt-local files')
  83. # ----------- Generate embedded python distribution -----------
  84. os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # *.pyc files double the size of the distribution
  85. get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
  86. latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version
  87. if bitness == '32':
  88. latest_dist_url += '-embed-win32.zip'
  89. else:
  90. latest_dist_url += '-embed-amd64.zip'
  91. # I've verified that all the dlls in the following are signed by Microsoft.
  92. # Using this because Microsoft only provides installers whose files can't be
  93. # extracted without a special tool.
  94. if bitness == '32':
  95. visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x86.7z'
  96. visual_c_runtime_sha256 = '2549eb4d2ce4cf3a87425ea01940f74368bf1cda378ef8a8a1f1a12ed59f1547'
  97. visual_c_name = 'vc15_(14.10.25017.0)_2017_x86.7z'
  98. visual_c_path_to_dlls = 'runtime_minimum/System'
  99. else:
  100. visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x64.7z'
  101. visual_c_runtime_sha256 = '4f00b824c37e1017a93fccbd5775e6ee54f824b6786f5730d257a87a3d9ce921'
  102. visual_c_name = 'vc15_(14.10.25017.0)_2017_x64.7z'
  103. visual_c_path_to_dlls = 'runtime_minimum/System64'
  104. download_if_not_exists('get-pip.py', get_pip_url)
  105. python_dist_name = 'python-dist-' + latest_version + '-' + bitness + '.zip'
  106. download_if_not_exists(python_dist_name, latest_dist_url)
  107. download_if_not_exists(visual_c_name,
  108. visual_c_runtime_url, sha256=visual_c_runtime_sha256)
  109. if os.path.exists('./python'):
  110. log('Removing old python distribution')
  111. shutil.rmtree('./python')
  112. log('Extracting python distribution')
  113. check(os.system(r'7z -y x -opython ' + python_dist_name))
  114. log('Executing get-pip.py')
  115. wine_run(['./python/python.exe', '-I', 'get-pip.py'])
  116. '''
  117. # Explanation of .pth, ._pth, and isolated mode
  118. ## Isolated mode
  119. We want to run in what is called isolated mode, given by the switch -I.
  120. This mode prevents the embedded python distribution from searching in
  121. global directories for imports
  122. For example, if a user has `C:\Python37` and the embedded distribution is
  123. the same version, importing something using the embedded distribution will
  124. search `C:\Python37\Libs\site-packages`. This is not desirable because it
  125. means I might forget to distribute a dependency if I have it installed
  126. globally and I don't see any import errors. It also means that an outdated
  127. package might override the one being distributed and cause other problems.
  128. Isolated mode also means global environment variables and registry
  129. entries will be ignored
  130. ## The trouble with isolated mode
  131. Isolated mode also prevents the current working directory (cwd) from
  132. being added to `sys.path`. `sys.path` is the list of directories python will
  133. search in for imports. In non-isolated mode this is automatically populated
  134. with the cwd, `site-packages`, the directory of the python executable, etc.
  135. # How to get the cwd into sys.path in isolated mode
  136. The hack to get this to work is to use a .pth file. Normally, these files
  137. are just an additional list of directories to be added to `sys.path`.
  138. However, they also allow arbitrary code execution on lines beginning with
  139. `import ` (see https://docs.python.org/3/library/site.html). So, we simply
  140. add `import sys; sys.path.insert(0, '')` to add the cwd to path. `''` is
  141. shorthand for the cwd. See https://bugs.python.org/issue33698#msg318272
  142. # ._pth files in the embedded distribution
  143. A python37._pth file is included in the embedded distribution. The presence
  144. of tis file causes the embedded distribution to always use isolated mode
  145. (which we want). They are like .pth files, except they do not allow the
  146. arbitrary code execution trick. In my experimentation, I found that they
  147. prevent .pth files from loading. So the ._pth file will have to be removed
  148. and replaced with a .pth. Isolated mode will have to be specified manually.
  149. '''
  150. log('Removing ._pth')
  151. major_release = latest_version.split('.')[1]
  152. os.remove(r'./python/python3' + major_release + '._pth')
  153. log('Adding path_fixes.pth')
  154. with open(r'./python/path_fixes.pth', 'w', encoding='utf-8') as f:
  155. f.write("import sys; sys.path.insert(0, '')\n")
  156. '''# python3x._pth file tells the python executable where to look for files
  157. # Need to add the directory where packages are installed,
  158. # and the parent directory (which is where the yt-local files are)
  159. major_release = latest_version.split('.')[1]
  160. with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f:
  161. f.write('.\\Lib\\site-packages\n')
  162. f.write('..\n')'''
  163. log('Inserting Microsoft C Runtime')
  164. check_subp(subprocess.run([r'7z', '-y', 'e', '-opython', visual_c_name, visual_c_path_to_dlls]))
  165. log('Installing dependencies')
  166. wine_run(['./python/python.exe', '-I', '-m', 'pip', 'install', '--no-compile', '-r', './requirements.txt'])
  167. log('Uninstalling unnecessary gevent stuff')
  168. wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'cffi', 'pycparser'])
  169. shutil.rmtree(r'./python/Lib/site-packages/gevent/tests')
  170. shutil.rmtree(r'./python/Lib/site-packages/gevent/testing')
  171. remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation
  172. log('Uninstalling pip and others')
  173. wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'pip', 'wheel'])
  174. log('Removing pyc files') # Have to do this because get-pip and some packages don't respect --no-compile
  175. remove_files_with_extensions(r'./python', ['.pyc'])
  176. log('Removing dist-info and __pycache__')
  177. for root, dirs, files in os.walk(r'./python'):
  178. for dir in dirs:
  179. if dir == '__pycache__' or dir.endswith('.dist-info'):
  180. shutil.rmtree(os.path.join(root, dir))
  181. '''log('Removing get-pip.py and zipped distribution')
  182. os.remove(r'.\get-pip.py')
  183. os.remove(r'.\latest-dist.zip')'''
  184. print()
  185. log('Finished generating python distribution')
  186. # ----------- Copy generated distribution into release folder -----------
  187. log('Copying python distribution into release folder')
  188. shutil.copytree(r'./python', r'./yt-local/python')
  189. # ----------- Create release zip -----------
  190. output_filename = 'yt-local-' + release_tag + '-' + suffix + '.zip'
  191. if os.path.exists('./' + output_filename):
  192. log('Removing previous zipped release')
  193. os.remove('./' + output_filename)
  194. log('Zipping release')
  195. check(os.system(r'7z -mx=9 a ' + output_filename + ' ./yt-local'))
  196. print('\n')
  197. log('Finished')