setup_cython.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504
  1. #
  2. # Cython patcher
  3. # v1.20
  4. #
  5. # Copyright (C) 2012-2019 Michael Buesch <m@bues.ch>
  6. #
  7. # This program is free software: you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation, either version 2 of the License, or
  10. # (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License
  18. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  19. #
  20. from __future__ import print_function
  21. import sys
  22. import os
  23. import platform
  24. import errno
  25. import shutil
  26. import hashlib
  27. import re
  28. WORKER_MEM_BYTES = 800 * 1024*1024
  29. WORKER_CPU_OVERCOMMIT = 2
  30. setupFileName = "setup.py"
  31. parallelBuild = False
  32. profileEnabled = False
  33. debugEnabled = False
  34. ext_modules = []
  35. CythonBuildExtension = None
  36. patchDirName = "cython_patched.%s-%s-%d.%d" % (
  37. platform.system().lower(),
  38. platform.machine().lower(),
  39. sys.version_info[0],
  40. sys.version_info[1])
  41. _Cython_Distutils_build_ext = None
  42. _cythonPossible = None
  43. _cythonBuildUnits = []
  44. _isWindows = os.name.lower() in {"nt", "ce"}
  45. _isPosix = os.name.lower() == "posix"
  46. def getSystemMemBytesCount():
  47. try:
  48. with open("/proc/meminfo", "rb") as fd:
  49. for line in fd.read().decode("UTF-8", "ignore").splitlines():
  50. if line.startswith("MemTotal:") and\
  51. line.endswith("kB"):
  52. kB = int(line.split()[1], 10)
  53. return kB * 1024
  54. except (OSError, IndexError, ValueError, UnicodeError) as e:
  55. pass
  56. if hasattr(os, "sysconf"):
  57. try:
  58. return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
  59. except ValueError as e:
  60. pass
  61. return None
  62. def makedirs(path, mode=0o755):
  63. try:
  64. os.makedirs(path, mode)
  65. except OSError as e:
  66. if e.errno == errno.EEXIST:
  67. return
  68. raise e
  69. def hashFile(path):
  70. if sys.version_info[0] < 3:
  71. ExpectedException = IOError
  72. else:
  73. ExpectedException = FileNotFoundError
  74. try:
  75. with open(path, "rb") as fd:
  76. return hashlib.sha1(fd.read()).hexdigest()
  77. except ExpectedException as e:
  78. return None
  79. def __fileopIfChanged(fromFile, toFile, fileops):
  80. toFileHash = hashFile(toFile)
  81. if toFileHash is not None:
  82. fromFileHash = hashFile(fromFile)
  83. if toFileHash == fromFileHash:
  84. return False
  85. makedirs(os.path.dirname(toFile))
  86. for fileop in fileops:
  87. fileop(fromFile, toFile)
  88. return True
  89. def removeFile(filename):
  90. try:
  91. os.unlink(filename)
  92. except OSError:
  93. pass
  94. def copyIfChanged(fromFile, toFile):
  95. fileops = []
  96. if _isWindows:
  97. fileops.append(lambda _fromFile, _toFile: removeFile(_toFile))
  98. fileops.append(shutil.copy2)
  99. return __fileopIfChanged(fromFile, toFile, fileops)
  100. def moveIfChanged(fromFile, toFile):
  101. fileops = []
  102. if _isWindows:
  103. fileops.append(lambda _fromFile, _toFile: removeFile(_toFile))
  104. fileops.append(os.rename)
  105. return __fileopIfChanged(fromFile, toFile, fileops)
  106. def makeDummyFile(path):
  107. if os.path.isfile(path):
  108. return
  109. print("creating dummy file '%s'" % path)
  110. makedirs(os.path.dirname(path))
  111. with open(path, "wb") as fd:
  112. fd.write("\n".encode("UTF-8"))
  113. def pyCythonPatchLine(line):
  114. return line
  115. def pyCythonPatch(fromFile, toFile):
  116. print("cython-patch: patching file '%s' to '%s'" %\
  117. (fromFile, toFile))
  118. tmpFile = toFile + ".TMP"
  119. makedirs(os.path.dirname(tmpFile))
  120. with open(fromFile, "rb") as infd,\
  121. open(tmpFile, "wb") as outfd:
  122. for line in infd.read().decode("UTF-8").splitlines(True):
  123. stripLine = line.strip()
  124. if stripLine.endswith("#@no-cython-patch"):
  125. outfd.write(line.encode("UTF-8"))
  126. continue
  127. # Replace import by cimport as requested by #+cimport
  128. if "#+cimport" in stripLine:
  129. line = line.replace("#+cimport", "#")
  130. line = re.sub(r'\bimport\b', "cimport", line)
  131. # Convert None to NULL
  132. if "#+NoneToNULL" in stripLine:
  133. line = line.replace("#+NoneToNULL", "#")
  134. line = re.sub(r'\bNone\b', "NULL", line)
  135. # Uncomment all lines containing #@cy
  136. def uncomment(line, removeStr):
  137. line = line.replace(removeStr, "")
  138. if line.startswith("#"):
  139. line = line[1:]
  140. if not line.endswith("\n"):
  141. line += "\n"
  142. return line
  143. if "#@cy-posix" in stripLine:
  144. if _isPosix:
  145. line = uncomment(line, "#@cy-posix")
  146. elif "#@cy-win" in stripLine:
  147. if _isWindows:
  148. line = uncomment(line, "#@cy-win")
  149. elif "#@cy2" in stripLine:
  150. if sys.version_info[0] < 3:
  151. line = uncomment(line, "#@cy2")
  152. elif "#@cy3" in stripLine:
  153. if sys.version_info[0] >= 3:
  154. line = uncomment(line, "#@cy3")
  155. elif "#@cy" in stripLine:
  156. line = uncomment(line, "#@cy")
  157. # Sprinkle magic cdef/cpdef, as requested by #+cdef/#+cpdef
  158. if "#+cdef-" in stripLine:
  159. # +cdef-foo-bar is the extended cdef patching.
  160. # It adds cdef and any additional characters to the
  161. # start of the line. Dashes are replaced with spaces.
  162. # Get the additional text
  163. idx = line.find("#+cdef-")
  164. cdefText = line[idx+2 : ]
  165. cdefText = cdefText.replace("-", " ").rstrip("\r\n")
  166. # Get the initial space length
  167. spaceCnt = 0
  168. while spaceCnt < len(line) and line[spaceCnt].isspace():
  169. spaceCnt += 1
  170. # Construct the new line
  171. line = line[ : spaceCnt] + cdefText + " " + line[spaceCnt : ]
  172. elif "#+cdef" in stripLine:
  173. # Simple cdef patching:
  174. # def -> cdef
  175. # class -> cdef class
  176. if stripLine.startswith("class"):
  177. line = re.sub(r'\bclass\b', "cdef class", line)
  178. else:
  179. line = re.sub(r'\bdef\b', "cdef", line)
  180. if "#+cpdef" in stripLine:
  181. # Simple cpdef patching:
  182. # def -> cpdef
  183. line = re.sub(r'\bdef\b', "cpdef", line)
  184. # Add likely()/unlikely() to if-conditions.
  185. for likely in ("likely", "unlikely"):
  186. if "#+" + likely in stripLine:
  187. line = re.sub(r'\bif\s(.*):', r'if ' + likely + r'(\1):', line)
  188. break
  189. # Add an "u" suffix to decimal and hexadecimal numbers.
  190. if "#+suffix-u" in line or "#+suffix-U" in line:
  191. line = re.sub(r'\b([0-9]+)\b', r'\1u', line)
  192. line = re.sub(r'\b(0x[0-9a-fA-F]+)\b', r'\1u', line)
  193. # Add an "LL" suffix to decimal and hexadecimal numbers.
  194. if "#+suffix-ll" in line or "#+suffix-LL" in line:
  195. line = re.sub(r'\b(\-?[0-9]+)\b', r'\1LL', line)
  196. line = re.sub(r'\b(0x[0-9a-fA-F]+)\b', r'\1LL', line)
  197. # Comment all lines containing #@nocy
  198. if "#@nocy" in stripLine:
  199. line = "#" + line
  200. # Comment all lines containing #@cyX
  201. # for the not matching version.
  202. if sys.version_info[0] < 3:
  203. if "#@cy3" in stripLine:
  204. line = "#" + line
  205. else:
  206. if "#@cy2" in stripLine:
  207. line = "#" + line
  208. # Comment all lines containing #@cy-posix/win
  209. # for the not matching platform.
  210. if _isPosix:
  211. if "#@cy-win" in stripLine:
  212. line = "#" + line
  213. elif _isWindows:
  214. if "#@cy-posix" in stripLine:
  215. line = "#" + line
  216. # Remove compat stuff
  217. line = line.replace("absolute_import,", "")
  218. line = pyCythonPatchLine(line)
  219. outfd.write(line.encode("UTF-8"))
  220. outfd.flush()
  221. if moveIfChanged(tmpFile, toFile):
  222. print("(updated)")
  223. else:
  224. os.unlink(tmpFile)
  225. class CythonBuildUnit(object):
  226. def __init__(self, cyModName, baseName, fromPy, fromPxd, toDir, toPyx, toPxd):
  227. self.cyModName = cyModName
  228. self.baseName = baseName
  229. self.fromPy = fromPy
  230. self.fromPxd = fromPxd
  231. self.toDir = toDir
  232. self.toPyx = toPyx
  233. self.toPxd = toPxd
  234. def patchCythonModules(buildDir):
  235. for unit in _cythonBuildUnits:
  236. makedirs(unit.toDir)
  237. makeDummyFile(os.path.join(unit.toDir, "__init__.py"))
  238. if unit.baseName == "__init__":
  239. # Copy and patch the package __init__.py
  240. toPy = os.path.join(buildDir, *unit.cyModName.split(".")) + ".py"
  241. pyCythonPatch(unit.fromPy, toPy)
  242. else:
  243. # Generate the .pyx
  244. pyCythonPatch(unit.fromPy, unit.toPyx)
  245. # Copy and patch the .pxd, if any
  246. if os.path.isfile(unit.fromPxd):
  247. pyCythonPatch(unit.fromPxd, unit.toPxd)
  248. def registerCythonModule(baseDir, sourceModName):
  249. global ext_modules
  250. global _cythonBuildUnits
  251. modDir = os.path.join(baseDir, sourceModName)
  252. # Make path to the cython patch-build-dir
  253. patchDir = os.path.join(baseDir, "build", patchDirName,
  254. ("%s_cython" % sourceModName))
  255. if not os.path.exists(os.path.join(baseDir, setupFileName)) or\
  256. not os.path.exists(modDir) or\
  257. not os.path.isdir(modDir):
  258. raise Exception("Wrong directory. "
  259. "Execute setup.py from within the main directory.")
  260. # Walk the module
  261. for dirpath, dirnames, filenames in os.walk(modDir):
  262. subpath = os.path.relpath(dirpath, modDir)
  263. if subpath == baseDir:
  264. subpath = ""
  265. dirpathList = dirpath.split(os.path.sep)
  266. if any(os.path.exists(os.path.sep.join(dirpathList[:i] + ["no_cython"]))
  267. for i in range(len(dirpathList) + 1)):
  268. # no_cython file exists. -> skip
  269. continue
  270. for filename in filenames:
  271. if filename.endswith(".py"):
  272. fromSuffix = ".py"
  273. elif filename.endswith(".pyx.in"):
  274. fromSuffix = ".pyx.in"
  275. else:
  276. continue
  277. baseName = filename[:-len(fromSuffix)] # Strip .py/.pyx.in
  278. fromPy = os.path.join(dirpath, baseName + fromSuffix)
  279. fromPxd = os.path.join(dirpath, baseName + ".pxd.in")
  280. toDir = os.path.join(patchDir, subpath)
  281. toPyx = os.path.join(toDir, baseName + ".pyx")
  282. toPxd = os.path.join(toDir, baseName + ".pxd")
  283. # Construct the new cython module name
  284. cyModName = [ "%s_cython" % sourceModName ]
  285. if subpath:
  286. cyModName.extend(subpath.split(os.sep))
  287. cyModName.append(baseName)
  288. cyModName = ".".join(cyModName)
  289. # Remember the filenames for the build
  290. unit = CythonBuildUnit(cyModName, baseName, fromPy, fromPxd,
  291. toDir, toPyx, toPxd)
  292. _cythonBuildUnits.append(unit)
  293. if baseName != "__init__":
  294. # Create a distutils Extension for the module
  295. extra_compile_args = []
  296. extra_link_args = []
  297. if not _isWindows:
  298. extra_compile_args.append("-Wall")
  299. extra_compile_args.append("-Wextra")
  300. #extra_compile_args.append("-Wcast-qual")
  301. extra_compile_args.append("-Wlogical-op")
  302. extra_compile_args.append("-Wpointer-arith")
  303. extra_compile_args.append("-Wundef")
  304. extra_compile_args.append("-Wno-cast-function-type")
  305. extra_compile_args.append("-Wno-maybe-uninitialized")
  306. extra_compile_args.append("-Wno-type-limits")
  307. if debugEnabled:
  308. # Enable debugging and UBSAN.
  309. extra_compile_args.append("-g3")
  310. extra_compile_args.append("-fsanitize=undefined")
  311. extra_compile_args.append("-fsanitize=float-divide-by-zero")
  312. extra_compile_args.append("-fsanitize=float-cast-overflow")
  313. extra_compile_args.append("-fno-sanitize-recover")
  314. extra_link_args.append("-lubsan")
  315. else:
  316. # Disable all debugging symbols.
  317. extra_compile_args.append("-g0")
  318. extra_link_args.append("-Wl,--strip-all")
  319. ext_modules.append(
  320. _Cython_Distutils_Extension(
  321. cyModName,
  322. [toPyx],
  323. cython_directives={
  324. # Enable profile hooks?
  325. "profile" : profileEnabled,
  326. "linetrace" : profileEnabled,
  327. # Warn about unused variables?
  328. "warn.unused" : False,
  329. # Set language version
  330. "language_level" : 2 if sys.version_info[0] < 3 else 3,
  331. },
  332. define_macros=[
  333. ("CYTHON_TRACE", str(int(profileEnabled))),
  334. ("CYTHON_TRACE_NOGIL", str(int(profileEnabled))),
  335. ],
  336. include_dirs=[
  337. os.path.join("libs", "cython_headers"),
  338. ],
  339. extra_compile_args=extra_compile_args,
  340. extra_link_args=extra_link_args
  341. )
  342. )
  343. def registerCythonModules(baseDir=None):
  344. if baseDir is None:
  345. baseDir = os.curdir
  346. for filename in os.listdir(baseDir):
  347. if os.path.isdir(os.path.join(baseDir, filename)) and\
  348. os.path.exists(os.path.join(baseDir, filename, "__init__.py")) and\
  349. not os.path.exists(os.path.join(baseDir, filename, "no_cython")):
  350. registerCythonModule(baseDir, filename)
  351. def cythonBuildPossible():
  352. global _cythonPossible
  353. if _cythonPossible is not None:
  354. return _cythonPossible
  355. _cythonPossible = False
  356. try:
  357. import Cython.Compiler.Options
  358. # Omit docstrings in cythoned modules.
  359. Cython.Compiler.Options.docstrings = False
  360. # Generate module exit cleanup code.
  361. Cython.Compiler.Options.generate_cleanup_code = True
  362. # Generate HTML outputs.
  363. Cython.Compiler.Options.annotate = True
  364. from Cython.Distutils import build_ext, Extension
  365. global _Cython_Distutils_build_ext
  366. global _Cython_Distutils_Extension
  367. _Cython_Distutils_build_ext = build_ext
  368. _Cython_Distutils_Extension = Extension
  369. except ImportError as e:
  370. print("WARNING: Could not build the CYTHON modules: "
  371. "%s" % str(e))
  372. print("--> Is Cython installed?")
  373. return False
  374. _cythonPossible = True
  375. return True
  376. if sys.version_info[0] < 3:
  377. # Cython2 build libraries need method pickling
  378. # for parallel build.
  379. def unpickle_method(fname, obj, cls):
  380. # Ignore MRO. We don't seem to inherit methods.
  381. return cls.__dict__[fname].__get__(obj, cls)
  382. def pickle_method(m):
  383. return unpickle_method, (m.im_func.__name__,
  384. m.im_self,
  385. m.im_class)
  386. import copy_reg, types
  387. copy_reg.pickle(types.MethodType, pickle_method, unpickle_method)
  388. def cyBuildWrapper(arg):
  389. # This function does the same thing as the for-loop-body
  390. # inside of Cython's build_ext.build_extensions() method.
  391. # It is called via multiprocessing to build extensions
  392. # in parallel.
  393. # Note that this might break, if Cython's build_extensions()
  394. # is changed and stuff is added to its for loop. Meh.
  395. self, ext = arg
  396. ext.sources = self.cython_sources(ext.sources, ext)
  397. self.build_extension(ext)
  398. if cythonBuildPossible():
  399. # Override Cython's build_ext class.
  400. class CythonBuildExtension(_Cython_Distutils_build_ext):
  401. class Error(Exception): pass
  402. def build_extension(self, ext):
  403. assert(not ext.name.endswith("__init__"))
  404. _Cython_Distutils_build_ext.build_extension(self, ext)
  405. def build_extensions(self):
  406. global parallelBuild
  407. # First patch the files, the run the build
  408. patchCythonModules(self.build_lib)
  409. if parallelBuild:
  410. # Run the parallel build, yay.
  411. try:
  412. self.check_extensions_list(self.extensions)
  413. # Calculate the number of worker processes to use.
  414. memBytes = getSystemMemBytesCount()
  415. if memBytes is None:
  416. raise self.Error("Unknown system memory size")
  417. print("System memory detected: %d MB" % (memBytes // (1024*1024)))
  418. memProcsMax = memBytes // WORKER_MEM_BYTES
  419. if memProcsMax < 2:
  420. raise self.Error("Not enough system memory")
  421. import multiprocessing
  422. numProcs = min(multiprocessing.cpu_count() + WORKER_CPU_OVERCOMMIT,
  423. memProcsMax)
  424. # Start the worker pool.
  425. print("Building in parallel with %d workers." % numProcs)
  426. from multiprocessing.pool import Pool
  427. Pool(numProcs).map(cyBuildWrapper,
  428. ((self, ext) for ext in self.extensions))
  429. except (OSError, self.Error) as e:
  430. # OSError might happen in a restricted
  431. # environment like chroot.
  432. print("WARNING: Parallel build "
  433. "disabled due to: %s" % str(e))
  434. parallelBuild = False
  435. if not parallelBuild:
  436. # Run the normal non-parallel build.
  437. _Cython_Distutils_build_ext.build_extensions(self)