scu_builders.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364
  1. """Functions used to generate scu build source files during build time"""
  2. import glob
  3. import math
  4. import os
  5. from pathlib import Path
  6. from methods import print_error
  7. base_folder_path = str(Path(__file__).parent) + "/"
  8. base_folder_only = os.path.basename(os.path.normpath(base_folder_path))
  9. _verbose = False # Set manually for debug prints
  10. _scu_folders = set()
  11. _max_includes_per_scu = 1024
  12. def clear_out_stale_files(output_folder, extension, fresh_files):
  13. output_folder = os.path.abspath(output_folder)
  14. # print("clear_out_stale_files from folder: " + output_folder)
  15. if not os.path.isdir(output_folder):
  16. # folder does not exist or has not been created yet,
  17. # no files to clearout. (this is not an error)
  18. return
  19. for file in glob.glob(output_folder + "/*." + extension):
  20. file = Path(file)
  21. if file not in fresh_files:
  22. # print("removed stale file: " + str(file))
  23. os.remove(file)
  24. def folder_not_found(folder):
  25. abs_folder = base_folder_path + folder + "/"
  26. return not os.path.isdir(abs_folder)
  27. def find_files_in_folder(folder, sub_folder, include_list, extension, sought_exceptions, found_exceptions):
  28. abs_folder = base_folder_path + folder + "/" + sub_folder
  29. if not os.path.isdir(abs_folder):
  30. print_error(f'SCU: "{abs_folder}" not found.')
  31. return include_list, found_exceptions
  32. os.chdir(abs_folder)
  33. sub_folder_slashed = ""
  34. if sub_folder != "":
  35. sub_folder_slashed = sub_folder + "/"
  36. for file in glob.glob("*." + extension):
  37. simple_name = Path(file).stem
  38. if file.endswith(".gen.cpp"):
  39. continue
  40. li = '#include "' + folder + "/" + sub_folder_slashed + file + '"'
  41. if simple_name not in sought_exceptions:
  42. include_list.append(li)
  43. else:
  44. found_exceptions.append(li)
  45. return include_list, found_exceptions
  46. def write_output_file(file_count, include_list, start_line, end_line, output_folder, output_filename_prefix, extension):
  47. output_folder = os.path.abspath(output_folder)
  48. if not os.path.isdir(output_folder):
  49. # create
  50. os.mkdir(output_folder)
  51. if not os.path.isdir(output_folder):
  52. print_error(f'SCU: "{output_folder}" could not be created.')
  53. return
  54. if _verbose:
  55. print("SCU: Creating folder: %s" % output_folder)
  56. file_text = ""
  57. for i in range(start_line, end_line):
  58. if i < len(include_list):
  59. line = include_list[i]
  60. li = line + "\n"
  61. file_text += li
  62. num_string = ""
  63. if file_count > 0:
  64. num_string = "_" + str(file_count)
  65. short_filename = output_filename_prefix + num_string + ".gen." + extension
  66. output_filename = output_folder + "/" + short_filename
  67. output_path = Path(output_filename)
  68. if not output_path.exists() or output_path.read_text() != file_text:
  69. if _verbose:
  70. print("SCU: Generating: %s" % short_filename)
  71. output_path.write_text(file_text, encoding="utf8")
  72. elif _verbose:
  73. print("SCU: Generation not needed for: " + short_filename)
  74. return output_path
  75. def write_exception_output_file(file_count, exception_string, output_folder, output_filename_prefix, extension):
  76. output_folder = os.path.abspath(output_folder)
  77. if not os.path.isdir(output_folder):
  78. print_error(f"SCU: {output_folder} does not exist.")
  79. return
  80. file_text = exception_string + "\n"
  81. num_string = ""
  82. if file_count > 0:
  83. num_string = "_" + str(file_count)
  84. short_filename = output_filename_prefix + "_exception" + num_string + ".gen." + extension
  85. output_filename = output_folder + "/" + short_filename
  86. output_path = Path(output_filename)
  87. if not output_path.exists() or output_path.read_text() != file_text:
  88. if _verbose:
  89. print("SCU: Generating: " + short_filename)
  90. output_path.write_text(file_text, encoding="utf8")
  91. elif _verbose:
  92. print("SCU: Generation not needed for: " + short_filename)
  93. return output_path
  94. def find_section_name(sub_folder):
  95. # Construct a useful name for the section from the path for debug logging
  96. section_path = os.path.abspath(base_folder_path + sub_folder) + "/"
  97. folders = []
  98. folder = ""
  99. for i in range(8):
  100. folder = os.path.dirname(section_path)
  101. folder = os.path.basename(folder)
  102. if folder == base_folder_only:
  103. break
  104. folders.append(folder)
  105. section_path += "../"
  106. section_path = os.path.abspath(section_path) + "/"
  107. section_name = ""
  108. for n in range(len(folders)):
  109. section_name += folders[len(folders) - n - 1]
  110. if n != (len(folders) - 1):
  111. section_name += "_"
  112. return section_name
  113. # "folders" is a list of folders to add all the files from to add to the SCU
  114. # "section (like a module)". The name of the scu file will be derived from the first folder
  115. # (thus e.g. scene/3d becomes scu_scene_3d.gen.cpp)
  116. # "includes_per_scu" limits the number of includes in a single scu file.
  117. # This allows the module to be built in several translation units instead of just 1.
  118. # This will usually be slower to compile but will use less memory per compiler instance, which
  119. # is most relevant in release builds.
  120. # "sought_exceptions" are a list of files (without extension) that contain
  121. # e.g. naming conflicts, and are therefore not suitable for the scu build.
  122. # These will automatically be placed in their own separate scu file,
  123. # which is slow like a normal build, but prevents the naming conflicts.
  124. # Ideally in these situations, the source code should be changed to prevent naming conflicts.
  125. # "extension" will usually be cpp, but can also be set to c (for e.g. third party libraries that use c)
  126. def process_folder(folders, sought_exceptions=[], includes_per_scu=0, extension="cpp"):
  127. if len(folders) == 0:
  128. return
  129. # Construct the filename prefix from the FIRST folder name
  130. # e.g. "scene_3d"
  131. out_filename = find_section_name(folders[0])
  132. found_includes = []
  133. found_exceptions = []
  134. main_folder = folders[0]
  135. abs_main_folder = base_folder_path + main_folder
  136. # Keep a record of all folders that have been processed for SCU,
  137. # this enables deciding what to do when we call "add_source_files()"
  138. global _scu_folders
  139. _scu_folders.add(main_folder)
  140. # main folder (first)
  141. found_includes, found_exceptions = find_files_in_folder(
  142. main_folder, "", found_includes, extension, sought_exceptions, found_exceptions
  143. )
  144. # sub folders
  145. for d in range(1, len(folders)):
  146. found_includes, found_exceptions = find_files_in_folder(
  147. main_folder, folders[d], found_includes, extension, sought_exceptions, found_exceptions
  148. )
  149. found_includes = sorted(found_includes)
  150. # calculate how many lines to write in each file
  151. total_lines = len(found_includes)
  152. # adjust number of output files according to whether DEV or release
  153. num_output_files = 1
  154. if includes_per_scu == 0:
  155. includes_per_scu = _max_includes_per_scu
  156. else:
  157. if includes_per_scu > _max_includes_per_scu:
  158. includes_per_scu = _max_includes_per_scu
  159. num_output_files = max(math.ceil(total_lines / float(includes_per_scu)), 1)
  160. lines_per_file = math.ceil(total_lines / float(num_output_files))
  161. lines_per_file = max(lines_per_file, 1)
  162. start_line = 0
  163. # These do not vary throughout the loop
  164. output_folder = abs_main_folder + "/scu/"
  165. output_filename_prefix = "scu_" + out_filename
  166. fresh_files = set()
  167. for file_count in range(0, num_output_files):
  168. end_line = start_line + lines_per_file
  169. # special case to cover rounding error in final file
  170. if file_count == (num_output_files - 1):
  171. end_line = len(found_includes)
  172. fresh_file = write_output_file(
  173. file_count, found_includes, start_line, end_line, output_folder, output_filename_prefix, extension
  174. )
  175. fresh_files.add(fresh_file)
  176. start_line = end_line
  177. # Write the exceptions each in their own scu gen file,
  178. # so they can effectively compile in "old style / normal build".
  179. for exception_count in range(len(found_exceptions)):
  180. fresh_file = write_exception_output_file(
  181. exception_count, found_exceptions[exception_count], output_folder, output_filename_prefix, extension
  182. )
  183. fresh_files.add(fresh_file)
  184. # Clear out any stale file (usually we will be overwriting if necessary,
  185. # but we want to remove any that are pre-existing that will not be
  186. # overwritten, so as to not compile anything stale).
  187. clear_out_stale_files(output_folder, extension, fresh_files)
  188. def generate_scu_files(max_includes_per_scu):
  189. print("=============================")
  190. print("Single Compilation Unit Build")
  191. print("=============================")
  192. global _max_includes_per_scu
  193. _max_includes_per_scu = max_includes_per_scu
  194. print("SCU: Generating build files... (max includes per SCU: %d)" % _max_includes_per_scu)
  195. curr_folder = os.path.abspath("./")
  196. # check we are running from the correct folder
  197. if folder_not_found("core") or folder_not_found("platform") or folder_not_found("scene"):
  198. raise RuntimeError("scu_builders.py must be run from the godot folder.")
  199. return
  200. process_folder(["core"])
  201. process_folder(["core/crypto"])
  202. process_folder(["core/debugger"])
  203. process_folder(["core/extension"])
  204. process_folder(["core/input"])
  205. process_folder(["core/io"])
  206. process_folder(["core/math"])
  207. process_folder(["core/object"])
  208. process_folder(["core/os"])
  209. process_folder(["core/string"])
  210. process_folder(["core/variant"], ["variant_utility"])
  211. process_folder(["drivers/unix"])
  212. process_folder(["drivers/png"])
  213. process_folder(["editor"], ["file_system_dock", "editor_resource_preview"], 32)
  214. process_folder(["editor/debugger"])
  215. process_folder(["editor/debugger/debug_adapter"])
  216. process_folder(["editor/export"])
  217. process_folder(["editor/gui"])
  218. process_folder(["editor/import"])
  219. process_folder(["editor/import/3d"])
  220. process_folder(["editor/plugins"])
  221. process_folder(["editor/plugins/gizmos"])
  222. process_folder(["editor/plugins/tiles"])
  223. process_folder(["platform/android/export"])
  224. process_folder(["platform/ios/export"])
  225. process_folder(["platform/linuxbsd/export"])
  226. process_folder(["platform/macos/export"])
  227. process_folder(["platform/web/export"])
  228. process_folder(["platform/windows/export"])
  229. process_folder(["modules/gltf"])
  230. process_folder(["modules/gltf/structures"])
  231. process_folder(["modules/gltf/editor"])
  232. process_folder(["modules/gltf/extensions"])
  233. process_folder(["modules/gltf/extensions/physics"])
  234. process_folder(["modules/navigation"])
  235. process_folder(["modules/webrtc"])
  236. process_folder(["modules/websocket"])
  237. process_folder(["modules/gridmap"])
  238. process_folder(["modules/multiplayer"])
  239. process_folder(["modules/multiplayer/editor"])
  240. process_folder(["modules/openxr"], ["register_types"])
  241. process_folder(["modules/openxr/action_map"])
  242. process_folder(["modules/openxr/editor"])
  243. process_folder(["modules/csg"])
  244. process_folder(["modules/gdscript"])
  245. process_folder(["modules/gdscript/editor"])
  246. process_folder(["modules/gdscript/language_server"])
  247. process_folder(["scene/2d"])
  248. process_folder(["scene/2d/physics"])
  249. process_folder(["scene/2d/physics/joints"])
  250. process_folder(["scene/3d"])
  251. process_folder(["scene/3d/physics"])
  252. process_folder(["scene/3d/physics/joints"])
  253. process_folder(["scene/animation"])
  254. process_folder(["scene/gui"])
  255. process_folder(["scene/main"])
  256. process_folder(["scene/resources"])
  257. process_folder(["scene/resources/2d"])
  258. process_folder(["scene/resources/3d"])
  259. process_folder(["servers"])
  260. process_folder(["servers/rendering"])
  261. process_folder(["servers/rendering/storage"])
  262. process_folder(["servers/rendering/renderer_rd"])
  263. process_folder(["servers/rendering/renderer_rd/effects"])
  264. process_folder(["servers/rendering/renderer_rd/environment"])
  265. process_folder(["servers/rendering/renderer_rd/storage_rd"])
  266. process_folder(["servers/physics_2d"])
  267. process_folder(["servers/physics_3d"])
  268. process_folder(["servers/physics_3d/joints"])
  269. process_folder(["servers/audio"])
  270. process_folder(["servers/audio/effects"])
  271. # Finally change back the path to the calling folder
  272. os.chdir(curr_folder)
  273. if _verbose:
  274. print("SCU: Processed folders: %s" % sorted(_scu_folders))
  275. return _scu_folders