slave_pack.py 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. # ##### BEGIN GPL LICENSE BLOCK #####
  2. #
  3. # This program is free software; you can redistribute it and/or
  4. # modify it under the terms of the GNU General Public License
  5. # as published by the Free Software Foundation; either version 2
  6. # of the License, or (at your option) any later version.
  7. #
  8. # This program is distributed in the hope that it will be useful,
  9. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. # GNU General Public License for more details.
  12. #
  13. # You should have received a copy of the GNU General Public License
  14. # along with this program; if not, write to the Free Software Foundation,
  15. # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16. #
  17. # ##### END GPL LICENSE BLOCK #####
  18. # <pep8 compliant>
  19. # Runs on buildbot slave, creating a release package using the build
  20. # system and zipping it into buildbot_upload.zip. This is then uploaded
  21. # to the master in the next buildbot step.
  22. import os
  23. import subprocess
  24. import sys
  25. import zipfile
  26. # get builder name
  27. if len(sys.argv) < 2:
  28. sys.stderr.write("Not enough arguments, expecting builder name\n")
  29. sys.exit(1)
  30. builder = sys.argv[1]
  31. # Never write branch if it is master.
  32. branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
  33. blender_dir = os.path.join('..', 'blender.git')
  34. build_dir = os.path.join('..', 'build', builder)
  35. install_dir = os.path.join('..', 'install', builder)
  36. buildbot_upload_zip = os.path.abspath(os.path.join(os.path.dirname(install_dir), "buildbot_upload.zip"))
  37. upload_filename = None # Name of the archive to be uploaded
  38. # (this is the name of archive which will appear on the
  39. # download page)
  40. upload_filepath = None # Filepath to be uploaded to the server
  41. # (this folder will be packed)
  42. def parse_header_file(filename, define):
  43. import re
  44. regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
  45. with open(filename, "r") as file:
  46. for l in file:
  47. match = regex.match(l)
  48. if match:
  49. return match.group(1)
  50. return None
  51. # Make sure install directory always exists
  52. if not os.path.exists(install_dir):
  53. os.makedirs(install_dir)
  54. def create_tar_bz2(src, dest, package_name):
  55. # One extra to remove leading os.sep when cleaning root for package_root
  56. ln = len(src) + 1
  57. flist = list()
  58. # Create list of tuples containing file and archive name
  59. for root, dirs, files in os.walk(src):
  60. package_root = os.path.join(package_name, root[ln:])
  61. flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files])
  62. import tarfile
  63. package = tarfile.open(dest, 'w:bz2')
  64. for entry in flist:
  65. package.add(entry[0], entry[1], recursive=False)
  66. package.close()
  67. if builder.find('cmake') != -1:
  68. # CMake
  69. if 'win' in builder or 'mac' in builder:
  70. os.chdir(build_dir)
  71. files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')]
  72. for f in files:
  73. os.remove(f)
  74. retcode = subprocess.call(['cpack', '-G', 'ZIP'])
  75. result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
  76. # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
  77. # in the CMake yet. For until then we do some magic here.
  78. tokens = result_file.split('-')
  79. blender_version = tokens[1].split('.')
  80. blender_full_version = '.'.join(blender_version[0:2])
  81. git_hash = tokens[2].split('.')[1]
  82. platform = builder.split('_')[0]
  83. if platform == 'mac':
  84. # Special exception for OSX
  85. platform = 'OSX-10.9-'
  86. if builder.endswith('x86_64_10_9_cmake'):
  87. platform += 'x86_64'
  88. if builder.endswith('vc2015'):
  89. platform += "-vc14"
  90. builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform)
  91. # NOTE: Blender 2.7 is already respected by blender_full_version.
  92. if branch != '' and branch != 'blender2.7':
  93. builderified_name = branch + "-" + builderified_name
  94. os.rename(result_file, "{}.zip".format(builderified_name))
  95. # create zip file
  96. try:
  97. if os.path.exists(buildbot_upload_zip):
  98. os.remove(buildbot_upload_zip)
  99. z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED)
  100. z.write("{}.zip".format(builderified_name))
  101. z.close()
  102. sys.exit(retcode)
  103. except Exception as ex:
  104. sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
  105. sys.exit(1)
  106. elif builder.startswith('linux_'):
  107. blender = os.path.join(install_dir, 'blender')
  108. buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
  109. blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h")
  110. # Get version information
  111. blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
  112. blender_version = "%d.%d" % (blender_version // 100, blender_version % 100)
  113. blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
  114. blender_glibc = builder.split('_')[1]
  115. command_prefix = []
  116. bits = 64
  117. blender_arch = 'x86_64'
  118. if blender_glibc == 'glibc224':
  119. if builder.endswith('x86_64_cmake'):
  120. chroot_name = 'buildbot_stretch_x86_64'
  121. elif builder.endswith('i686_cmake'):
  122. chroot_name = 'buildbot_stretch_i686'
  123. bits = 32
  124. blender_arch = 'i686'
  125. command_prefix = ['schroot', '-c', chroot_name, '--']
  126. elif blender_glibc == 'glibc217':
  127. command_prefix = ['scl', 'enable', 'devtoolset-6', '--']
  128. # Strip all unused symbols from the binaries
  129. print("Stripping binaries...")
  130. subprocess.call(command_prefix + ['strip', '--strip-all', blender])
  131. print("Stripping python...")
  132. py_target = os.path.join(install_dir, blender_version)
  133. subprocess.call(command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';'])
  134. # Copy all specific files which are too specific to be copied by
  135. # the CMake rules themselves
  136. print("Copying extra scripts and libs...")
  137. extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
  138. mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
  139. software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
  140. icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
  141. os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
  142. os.system('cp %s %s' % (software_gl, install_dir))
  143. os.system('cp -r %s %s' % (icons, install_dir))
  144. os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
  145. # Construct archive name
  146. package_name = 'blender-%s-%s-linux-%s-%s' % (blender_version,
  147. blender_hash,
  148. blender_glibc,
  149. blender_arch)
  150. # NOTE: Blender 2.7 is already respected by blender_full_version.
  151. if branch != '' and branch != 'blender2.7':
  152. package_name = branch + "-" + package_name
  153. upload_filename = package_name + ".tar.bz2"
  154. print("Creating .tar.bz2 archive")
  155. upload_filepath = install_dir + '.tar.bz2'
  156. create_tar_bz2(install_dir, upload_filepath, package_name)
  157. else:
  158. print("Unknown building system")
  159. sys.exit(1)
  160. if upload_filepath is None:
  161. # clean release directory if it already exists
  162. release_dir = 'release'
  163. if os.path.exists(release_dir):
  164. for f in os.listdir(release_dir):
  165. if os.path.isfile(os.path.join(release_dir, f)):
  166. os.remove(os.path.join(release_dir, f))
  167. # create release package
  168. try:
  169. subprocess.call(['make', 'package_archive'])
  170. except Exception as ex:
  171. sys.stderr.write('Make package release failed' + str(ex) + '\n')
  172. sys.exit(1)
  173. # find release directory, must exist this time
  174. if not os.path.exists(release_dir):
  175. sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
  176. sys.exit(1)
  177. # find release package
  178. file = None
  179. filepath = None
  180. for f in os.listdir(release_dir):
  181. rf = os.path.join(release_dir, f)
  182. if os.path.isfile(rf) and f.startswith('blender'):
  183. file = f
  184. filepath = rf
  185. if not file:
  186. sys.stderr.write("Failed to find release package.\n")
  187. sys.exit(1)
  188. upload_filename = file
  189. upload_filepath = filepath
  190. # create zip file
  191. try:
  192. upload_zip = os.path.join(buildbot_upload_zip)
  193. if os.path.exists(upload_zip):
  194. os.remove(upload_zip)
  195. z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
  196. z.write(upload_filepath, arcname=upload_filename)
  197. z.close()
  198. except Exception as ex:
  199. sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
  200. sys.exit(1)