setup_cython.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479
  1. #
  2. # Cython patcher
  3. #
  4. # Copyright (C) 2012-2024 Michael Buesch <m@bues.ch>
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation, either version 2 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. #
  19. from __future__ import print_function
  20. import sys
  21. import os
  22. import platform
  23. import errno
  24. import shutil
  25. import hashlib
  26. import re
  27. WORKER_MEM_BYTES = 800 * 1024*1024
  28. WORKER_CPU_OVERCOMMIT = 2
  29. setupFileName = "setup.py"
  30. parallelBuild = False
  31. profileEnabled = False
  32. debugEnabled = False
  33. ext_modules = []
  34. CythonBuildExtension = None
  35. patchDirName = "cython_patched.%s-%s-%d.%d" % (
  36. platform.system().lower(),
  37. platform.machine().lower(),
  38. sys.version_info[0],
  39. sys.version_info[1])
  40. _Cython_Distutils_build_ext = None
  41. _cythonPossible = None
  42. _cythonBuildUnits = []
  43. _isWindows = os.name.lower() in {"nt", "ce"}
  44. _isPosix = os.name.lower() == "posix"
  45. def getSystemMemBytesCount():
  46. try:
  47. with open("/proc/meminfo", "rb") as fd:
  48. for line in fd.read().decode("UTF-8", "ignore").splitlines():
  49. if line.startswith("MemTotal:") and\
  50. line.endswith("kB"):
  51. kB = int(line.split()[1], 10)
  52. return kB * 1024
  53. except (OSError, IndexError, ValueError, UnicodeError) as e:
  54. pass
  55. if hasattr(os, "sysconf"):
  56. try:
  57. return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
  58. except ValueError as e:
  59. pass
  60. return None
  61. def makedirs(path, mode=0o755):
  62. try:
  63. os.makedirs(path, mode)
  64. except OSError as e:
  65. if e.errno == errno.EEXIST:
  66. return
  67. raise e
  68. def hashFile(path):
  69. try:
  70. with open(path, "rb") as fd:
  71. return hashlib.sha1(fd.read()).hexdigest()
  72. except FileNotFoundError as e:
  73. return None
  74. def __fileopIfChanged(fromFile, toFile, fileops):
  75. toFileHash = hashFile(toFile)
  76. if toFileHash is not None:
  77. fromFileHash = hashFile(fromFile)
  78. if toFileHash == fromFileHash:
  79. return False
  80. makedirs(os.path.dirname(toFile))
  81. for fileop in fileops:
  82. fileop(fromFile, toFile)
  83. return True
  84. def removeFile(filename):
  85. try:
  86. os.unlink(filename)
  87. except OSError:
  88. pass
  89. def copyIfChanged(fromFile, toFile):
  90. fileops = []
  91. if _isWindows:
  92. fileops.append(lambda _fromFile, _toFile: removeFile(_toFile))
  93. fileops.append(shutil.copy2)
  94. return __fileopIfChanged(fromFile, toFile, fileops)
  95. def moveIfChanged(fromFile, toFile):
  96. fileops = []
  97. if _isWindows:
  98. fileops.append(lambda _fromFile, _toFile: removeFile(_toFile))
  99. fileops.append(os.rename)
  100. return __fileopIfChanged(fromFile, toFile, fileops)
  101. def makeDummyFile(path):
  102. if os.path.isfile(path):
  103. return
  104. print("creating dummy file '%s'" % path)
  105. makedirs(os.path.dirname(path))
  106. with open(path, "wb") as fd:
  107. fd.write("\n".encode("UTF-8"))
  108. def pyCythonPatchLine(line):
  109. return line
  110. def pyCythonPatch(fromFile, toFile):
  111. print("cython-patch: patching file '%s' to '%s'" %\
  112. (fromFile, toFile))
  113. tmpFile = toFile + ".TMP"
  114. makedirs(os.path.dirname(tmpFile))
  115. with open(fromFile, "rb") as infd,\
  116. open(tmpFile, "wb") as outfd:
  117. for line in infd.read().decode("UTF-8").splitlines(True):
  118. stripLine = line.strip()
  119. if stripLine.endswith("#@no-cython-patch"):
  120. outfd.write(line.encode("UTF-8"))
  121. continue
  122. # Replace import by cimport as requested by #+cimport
  123. if "#+cimport" in stripLine:
  124. line = line.replace("#+cimport", "#")
  125. line = re.sub(r'\bimport\b', "cimport", line)
  126. # Convert None to NULL
  127. if "#+NoneToNULL" in stripLine:
  128. line = line.replace("#+NoneToNULL", "#")
  129. line = re.sub(r'\bNone\b', "NULL", line)
  130. # Uncomment all lines containing #@cy
  131. def uncomment(line, removeStr):
  132. line = line.replace(removeStr, "")
  133. if line.startswith("#"):
  134. line = line[1:]
  135. if not line.endswith("\n"):
  136. line += "\n"
  137. return line
  138. if "#@cy-posix" in stripLine:
  139. if _isPosix:
  140. line = uncomment(line, "#@cy-posix")
  141. elif "#@cy-win" in stripLine:
  142. if _isWindows:
  143. line = uncomment(line, "#@cy-win")
  144. elif "#@cy" in stripLine:
  145. line = uncomment(line, "#@cy")
  146. # Sprinkle magic cdef/cpdef, as requested by #+cdef/#+cpdef
  147. if "#+cdef-" in stripLine:
  148. # +cdef-foo-bar is the extended cdef patching.
  149. # It adds cdef and any additional characters to the
  150. # start of the line. Dashes are replaced with spaces.
  151. # Get the additional text
  152. idx = line.find("#+cdef-")
  153. cdefText = line[idx+2 : ]
  154. cdefText = cdefText.replace("-", " ").rstrip("\r\n")
  155. # Get the initial space length
  156. spaceCnt = 0
  157. while spaceCnt < len(line) and line[spaceCnt].isspace():
  158. spaceCnt += 1
  159. # Construct the new line
  160. line = line[ : spaceCnt] + cdefText + " " + line[spaceCnt : ]
  161. elif "#+cdef" in stripLine:
  162. # Simple cdef patching:
  163. # def -> cdef
  164. # class -> cdef class
  165. if stripLine.startswith("class"):
  166. line = re.sub(r'\bclass\b', "cdef class", line)
  167. else:
  168. line = re.sub(r'\bdef\b', "cdef", line)
  169. if "#+cpdef" in stripLine:
  170. # Simple cpdef patching:
  171. # def -> cpdef
  172. line = re.sub(r'\bdef\b', "cpdef", line)
  173. # Add likely()/unlikely() to if-conditions.
  174. for likely in ("likely", "unlikely"):
  175. if "#+" + likely in stripLine:
  176. line = re.sub(r'\bif\s(.*):', r'if ' + likely + r'(\1):', line)
  177. break
  178. # Add an "u" suffix to decimal and hexadecimal numbers.
  179. if "#+suffix-u" in line or "#+suffix-U" in line:
  180. line = re.sub(r'\b([0-9]+)\b', r'\1u', line)
  181. line = re.sub(r'\b(0x[0-9a-fA-F]+)\b', r'\1u', line)
  182. # Add an "LL" suffix to decimal and hexadecimal numbers.
  183. if "#+suffix-ll" in line or "#+suffix-LL" in line:
  184. line = re.sub(r'\b(\-?[0-9]+)\b', r'\1LL', line)
  185. line = re.sub(r'\b(0x[0-9a-fA-F]+)\b', r'\1LL', line)
  186. # Comment all lines containing #@nocy
  187. if "#@nocy" in stripLine:
  188. line = "#" + line
  189. # Comment all lines containing #@cy-posix/win
  190. # for the not matching platform.
  191. if _isPosix:
  192. if "#@cy-win" in stripLine:
  193. line = "#" + line
  194. elif _isWindows:
  195. if "#@cy-posix" in stripLine:
  196. line = "#" + line
  197. # Remove compat stuff
  198. line = line.replace("absolute_import,", "")
  199. line = pyCythonPatchLine(line)
  200. outfd.write(line.encode("UTF-8"))
  201. outfd.flush()
  202. if moveIfChanged(tmpFile, toFile):
  203. print("(updated)")
  204. else:
  205. os.unlink(tmpFile)
  206. class CythonBuildUnit(object):
  207. def __init__(self, cyModName, baseName, fromPy, fromPxd, toDir, toPyx, toPxd):
  208. self.cyModName = cyModName
  209. self.baseName = baseName
  210. self.fromPy = fromPy
  211. self.fromPxd = fromPxd
  212. self.toDir = toDir
  213. self.toPyx = toPyx
  214. self.toPxd = toPxd
  215. def patchCythonModules(buildDir):
  216. for unit in _cythonBuildUnits:
  217. makedirs(unit.toDir)
  218. makeDummyFile(os.path.join(unit.toDir, "__init__.py"))
  219. if unit.baseName == "__init__":
  220. # Copy and patch the package __init__.py
  221. toPy = os.path.join(buildDir, *unit.cyModName.split(".")) + ".py"
  222. pyCythonPatch(unit.fromPy, toPy)
  223. else:
  224. # Generate the .pyx
  225. pyCythonPatch(unit.fromPy, unit.toPyx)
  226. # Copy and patch the .pxd, if any
  227. if os.path.isfile(unit.fromPxd):
  228. pyCythonPatch(unit.fromPxd, unit.toPxd)
  229. def registerCythonModule(baseDir, sourceModName):
  230. global ext_modules
  231. global _cythonBuildUnits
  232. modDir = os.path.join(baseDir, sourceModName)
  233. # Make path to the cython patch-build-dir
  234. patchDir = os.path.join(baseDir, "build", patchDirName,
  235. ("%s_cython" % sourceModName))
  236. if not os.path.exists(os.path.join(baseDir, setupFileName)) or\
  237. not os.path.exists(modDir) or\
  238. not os.path.isdir(modDir):
  239. raise Exception("Wrong directory. "
  240. "Execute setup.py from within the main directory.")
  241. # Walk the module
  242. for dirpath, dirnames, filenames in os.walk(modDir):
  243. subpath = os.path.relpath(dirpath, modDir)
  244. if subpath == baseDir:
  245. subpath = ""
  246. dirpathList = dirpath.split(os.path.sep)
  247. if any(os.path.exists(os.path.sep.join(dirpathList[:i] + ["no_cython"]))
  248. for i in range(len(dirpathList) + 1)):
  249. # no_cython file exists. -> skip
  250. continue
  251. for filename in filenames:
  252. if filename.endswith(".py"):
  253. fromSuffix = ".py"
  254. elif filename.endswith(".pyx.in"):
  255. fromSuffix = ".pyx.in"
  256. else:
  257. continue
  258. baseName = filename[:-len(fromSuffix)] # Strip .py/.pyx.in
  259. fromPy = os.path.join(dirpath, baseName + fromSuffix)
  260. fromPxd = os.path.join(dirpath, baseName + ".pxd.in")
  261. toDir = os.path.join(patchDir, subpath)
  262. toPyx = os.path.join(toDir, baseName + ".pyx")
  263. toPxd = os.path.join(toDir, baseName + ".pxd")
  264. # Construct the new cython module name
  265. cyModName = [ "%s_cython" % sourceModName ]
  266. if subpath:
  267. cyModName.extend(subpath.split(os.sep))
  268. cyModName.append(baseName)
  269. cyModName = ".".join(cyModName)
  270. # Remember the filenames for the build
  271. unit = CythonBuildUnit(cyModName, baseName, fromPy, fromPxd,
  272. toDir, toPyx, toPxd)
  273. _cythonBuildUnits.append(unit)
  274. if baseName != "__init__":
  275. # Create a distutils Extension for the module
  276. extra_compile_args = []
  277. extra_link_args = []
  278. if not _isWindows:
  279. extra_compile_args.append("-Wall")
  280. extra_compile_args.append("-Wextra")
  281. extra_compile_args.append("-Wcast-qual")
  282. extra_compile_args.append("-Wlogical-op")
  283. extra_compile_args.append("-Wpointer-arith")
  284. extra_compile_args.append("-Wundef")
  285. extra_compile_args.append("-Wno-cast-function-type")
  286. extra_compile_args.append("-Wno-maybe-uninitialized")
  287. extra_compile_args.append("-Wno-type-limits")
  288. if debugEnabled:
  289. # Enable debugging and UBSAN.
  290. extra_compile_args.append("-g3")
  291. extra_compile_args.append("-fsanitize=undefined")
  292. extra_compile_args.append("-fsanitize=float-divide-by-zero")
  293. extra_compile_args.append("-fsanitize=float-cast-overflow")
  294. extra_compile_args.append("-fno-sanitize-recover")
  295. extra_link_args.append("-lubsan")
  296. else:
  297. # Disable all debugging symbols.
  298. extra_compile_args.append("-g0")
  299. extra_link_args.append("-Wl,--strip-all")
  300. ext_modules.append(
  301. _Cython_Distutils_Extension(
  302. cyModName,
  303. [toPyx],
  304. cython_directives={
  305. # Enable profile hooks?
  306. "profile" : profileEnabled,
  307. "linetrace" : profileEnabled,
  308. # Warn about unused variables?
  309. "warn.unused" : False,
  310. # Set language version
  311. "language_level" : 3,
  312. },
  313. define_macros=[
  314. ("CYTHON_TRACE", str(int(profileEnabled))),
  315. ("CYTHON_TRACE_NOGIL", str(int(profileEnabled))),
  316. ],
  317. include_dirs=[
  318. os.path.join("libs", "cython_headers"),
  319. ],
  320. extra_compile_args=extra_compile_args,
  321. extra_link_args=extra_link_args
  322. )
  323. )
  324. def registerCythonModules(baseDir=None):
  325. if baseDir is None:
  326. baseDir = os.curdir
  327. for filename in os.listdir(baseDir):
  328. if os.path.isdir(os.path.join(baseDir, filename)) and\
  329. os.path.exists(os.path.join(baseDir, filename, "__init__.py")) and\
  330. not os.path.exists(os.path.join(baseDir, filename, "no_cython")):
  331. registerCythonModule(baseDir, filename)
  332. def cythonBuildPossible():
  333. global _cythonPossible
  334. if _cythonPossible is not None:
  335. return _cythonPossible
  336. _cythonPossible = False
  337. if sys.version_info[0] < 3:
  338. print("WARNING: Could not build the CYTHON modules: "
  339. "Cython 2 not supported. Please use Cython 3.")
  340. return False
  341. try:
  342. import Cython.Compiler.Options
  343. # Omit docstrings in cythoned modules.
  344. Cython.Compiler.Options.docstrings = False
  345. # Generate module exit cleanup code.
  346. Cython.Compiler.Options.generate_cleanup_code = True
  347. # Generate HTML outputs.
  348. Cython.Compiler.Options.annotate = True
  349. from Cython.Distutils import build_ext, Extension
  350. global _Cython_Distutils_build_ext
  351. global _Cython_Distutils_Extension
  352. _Cython_Distutils_build_ext = build_ext
  353. _Cython_Distutils_Extension = Extension
  354. except ImportError as e:
  355. print("WARNING: Could not build the CYTHON modules: "
  356. "%s" % str(e))
  357. print("--> Is Cython installed?")
  358. return False
  359. _cythonPossible = True
  360. return True
  361. def cyBuildWrapper(arg):
  362. # This function does the same thing as the for-loop-body
  363. # inside of Cython's build_ext.build_extensions() method.
  364. # It is called via multiprocessing to build extensions
  365. # in parallel.
  366. # Note that this might break, if Cython's build_extensions()
  367. # is changed and stuff is added to its for loop. Meh.
  368. self, ext = arg
  369. ext.sources = self.cython_sources(ext.sources, ext)
  370. self.build_extension(ext)
  371. if cythonBuildPossible():
  372. # Override Cython's build_ext class.
  373. class CythonBuildExtension(_Cython_Distutils_build_ext):
  374. class Error(Exception): pass
  375. def build_extension(self, ext):
  376. assert(not ext.name.endswith("__init__"))
  377. _Cython_Distutils_build_ext.build_extension(self, ext)
  378. def build_extensions(self):
  379. global parallelBuild
  380. # First patch the files, the run the build
  381. patchCythonModules(self.build_lib)
  382. # FIXME: Parallel build currently does not work on Cython 3.x
  383. parallelBuild = False
  384. if parallelBuild:
  385. # Run the parallel build, yay.
  386. try:
  387. self.check_extensions_list(self.extensions)
  388. # Calculate the number of worker processes to use.
  389. memBytes = getSystemMemBytesCount()
  390. if memBytes is None:
  391. raise self.Error("Unknown system memory size")
  392. print("System memory detected: %d MB" % (memBytes // (1024*1024)))
  393. memProcsMax = memBytes // WORKER_MEM_BYTES
  394. if memProcsMax < 2:
  395. raise self.Error("Not enough system memory")
  396. import multiprocessing
  397. numProcs = min(multiprocessing.cpu_count() + WORKER_CPU_OVERCOMMIT,
  398. memProcsMax)
  399. # Start the worker pool.
  400. print("Building in parallel with %d workers." % numProcs)
  401. from multiprocessing.pool import Pool
  402. Pool(numProcs).map(cyBuildWrapper,
  403. ((self, ext) for ext in self.extensions))
  404. except (OSError, self.Error) as e:
  405. # OSError might happen in a restricted
  406. # environment like chroot.
  407. print("WARNING: Parallel build "
  408. "disabled due to: %s" % str(e))
  409. parallelBuild = False
  410. if not parallelBuild:
  411. # Run the normal non-parallel build.
  412. _Cython_Distutils_build_ext.build_extensions(self)