setup_cython.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. #
  2. # Cython patcher
  3. # v1.10
  4. #
  5. # Copyright (C) 2012-2018 Michael Buesch <m@bues.ch>
  6. #
  7. # This program is free software: you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation, either version 2 of the License, or
  10. # (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. # GNU General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License
  18. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  19. #
  20. from __future__ import print_function
  21. import sys
  22. import os
  23. import platform
  24. import errno
  25. import shutil
  26. import hashlib
  27. import re
  28. parallelBuild = False
  29. profileEnabled = False
  30. ext_modules = []
  31. CythonBuildExtension = None
  32. _Cython_Distutils_build_ext = None
  33. _cythonPossible = None
  34. _cythonBuildUnits = []
  35. def makedirs(path, mode=0o755):
  36. try:
  37. os.makedirs(path, mode)
  38. except OSError as e:
  39. if e.errno == errno.EEXIST:
  40. return
  41. raise e
  42. def hashFile(path):
  43. if sys.version_info[0] < 3:
  44. ExpectedException = IOError
  45. else:
  46. ExpectedException = FileNotFoundError
  47. try:
  48. with open(path, "rb") as fd:
  49. return hashlib.sha1(fd.read()).hexdigest()
  50. except ExpectedException as e:
  51. return None
  52. def __fileopIfChanged(fromFile, toFile, fileop):
  53. toFileHash = hashFile(toFile)
  54. if toFileHash is not None:
  55. fromFileHash = hashFile(fromFile)
  56. if toFileHash == fromFileHash:
  57. return False
  58. makedirs(os.path.dirname(toFile))
  59. fileop(fromFile, toFile)
  60. return True
  61. def copyIfChanged(fromFile, toFile):
  62. return __fileopIfChanged(fromFile, toFile, shutil.copy2)
  63. def moveIfChanged(fromFile, toFile):
  64. return __fileopIfChanged(fromFile, toFile, os.rename)
  65. def makeDummyFile(path):
  66. if os.path.isfile(path):
  67. return
  68. print("creating dummy file '%s'" % path)
  69. makedirs(os.path.dirname(path))
  70. with open(path, "wb") as fd:
  71. fd.write("\n".encode("UTF-8"))
  72. def pyCythonPatchLine(line):
  73. return line
  74. def pyCythonPatch(fromFile, toFile):
  75. print("cython-patch: patching file '%s' to '%s'" %\
  76. (fromFile, toFile))
  77. tmpFile = toFile + ".TMP"
  78. makedirs(os.path.dirname(tmpFile))
  79. with open(fromFile, "rb") as infd,\
  80. open(tmpFile, "wb") as outfd:
  81. for line in infd.read().decode("UTF-8").splitlines(True):
  82. stripLine = line.strip()
  83. if stripLine.endswith("#<no-cython-patch"):
  84. outfd.write(line.encode("UTF-8"))
  85. continue
  86. # Replace import by cimport as requested by #+cimport
  87. if "#+cimport" in stripLine:
  88. line = line.replace("#+cimport", "#")
  89. line = re.sub(r'\bimport\b', "cimport", line)
  90. # Convert None to NULL
  91. if "#@cy-NoneToNULL" in stripLine:
  92. line = line.replace("#@cy-NoneToNULL", "#")
  93. line = re.sub(r'\bNone\b', "NULL", line)
  94. # Uncomment all lines containing #@cy
  95. def uncomment(line, removeStr):
  96. line = line.replace(removeStr, "")
  97. if line.startswith("#"):
  98. line = line[1:]
  99. if not line.endswith("\n"):
  100. line += "\n"
  101. return line
  102. if "#@cy" in stripLine and\
  103. not "#@cy2" in stripLine and\
  104. not "#@cy3" in stripLine:
  105. line = uncomment(line, "#@cy")
  106. if sys.version_info[0] < 3:
  107. if "#@cy2" in stripLine:
  108. line = uncomment(line, "#@cy2")
  109. else:
  110. if "#@cy3" in stripLine:
  111. line = uncomment(line, "#@cy3")
  112. # Sprinkle magic cdef/cpdef, as requested by #+cdef/#+cpdef
  113. if "#+cdef-" in stripLine:
  114. # +cdef-foo-bar is the extended cdef patching.
  115. # It adds cdef and any additional characters to the
  116. # start of the line. Dashes are replaced with spaces.
  117. # Get the additional text
  118. idx = line.find("#+cdef-")
  119. cdefText = line[idx+2 : ]
  120. cdefText = cdefText.replace("-", " ").rstrip("\r\n")
  121. # Get the initial space length
  122. spaceCnt = 0
  123. while spaceCnt < len(line) and line[spaceCnt].isspace():
  124. spaceCnt += 1
  125. # Construct the new line
  126. line = line[ : spaceCnt] + cdefText + " " + line[spaceCnt : ]
  127. elif "#+cdef" in stripLine:
  128. # Simple cdef patching:
  129. # def -> cdef
  130. # class -> cdef class
  131. if stripLine.startswith("class"):
  132. line = re.sub(r'\bclass\b', "cdef class", line)
  133. else:
  134. line = re.sub(r'\bdef\b', "cdef", line)
  135. if "#+cpdef" in stripLine:
  136. # Simple cpdef patching:
  137. # def -> cpdef
  138. line = re.sub(r'\bdef\b', "cpdef", line)
  139. # Comment all lines containing #@nocy
  140. # or #@cyX for the not matching version.
  141. if "#@nocy" in stripLine:
  142. line = "#" + line
  143. if sys.version_info[0] < 3:
  144. if "#@cy3" in stripLine:
  145. line = "#" + line
  146. else:
  147. if "#@cy2" in stripLine:
  148. line = "#" + line
  149. # Remove compat stuff
  150. line = line.replace("absolute_import,", "")
  151. line = pyCythonPatchLine(line)
  152. outfd.write(line.encode("UTF-8"))
  153. outfd.flush()
  154. if moveIfChanged(tmpFile, toFile):
  155. print("(updated)")
  156. else:
  157. os.unlink(tmpFile)
  158. class CythonBuildUnit(object):
  159. def __init__(self, cyModName, baseName, fromPy, fromPxd, toDir, toPyx, toPxd):
  160. self.cyModName = cyModName
  161. self.baseName = baseName
  162. self.fromPy = fromPy
  163. self.fromPxd = fromPxd
  164. self.toDir = toDir
  165. self.toPyx = toPyx
  166. self.toPxd = toPxd
  167. def patchCythonModules(buildDir):
  168. for unit in _cythonBuildUnits:
  169. makedirs(unit.toDir)
  170. makeDummyFile(os.path.join(unit.toDir, "__init__.py"))
  171. if unit.baseName == "__init__":
  172. # Copy and patch the package __init__.py
  173. toPy = os.path.join(buildDir, *unit.cyModName.split(".")) + ".py"
  174. pyCythonPatch(unit.fromPy, toPy)
  175. else:
  176. # Generate the .pyx
  177. pyCythonPatch(unit.fromPy, unit.toPyx)
  178. # Copy and patch the .pxd, if any
  179. if os.path.isfile(unit.fromPxd):
  180. pyCythonPatch(unit.fromPxd, unit.toPxd)
  181. def registerCythonModule(baseDir, sourceModName):
  182. global ext_modules
  183. global _cythonBuildUnits
  184. modDir = os.path.join(baseDir, sourceModName)
  185. # Make path to the cython patch-build-dir
  186. patchDir = os.path.join(baseDir, "build",
  187. "cython_patched.%s-%s-%d.%d" %\
  188. (platform.system().lower(),
  189. platform.machine().lower(),
  190. sys.version_info[0], sys.version_info[1]),
  191. "%s_cython" % sourceModName
  192. )
  193. if not os.path.exists(os.path.join(baseDir, "setup.py")) or\
  194. not os.path.exists(modDir) or\
  195. not os.path.isdir(modDir):
  196. raise Exception("Wrong directory. "
  197. "Execute setup.py from within the main directory.")
  198. # Walk the module
  199. for dirpath, dirnames, filenames in os.walk(modDir):
  200. subpath = os.path.relpath(dirpath, modDir)
  201. if subpath == baseDir:
  202. subpath = ""
  203. dirpathList = dirpath.split(os.path.sep)
  204. if any(os.path.exists(os.path.sep.join(dirpathList[:i] + ["no_cython"]))
  205. for i in range(len(dirpathList) + 1)):
  206. # no_cython file exists. -> skip
  207. continue
  208. for filename in filenames:
  209. if filename.endswith(".py"):
  210. fromSuffix = ".py"
  211. elif filename.endswith(".pyx.in"):
  212. fromSuffix = ".pyx.in"
  213. else:
  214. continue
  215. baseName = filename[:-len(fromSuffix)] # Strip .py/.pyx.in
  216. fromPy = os.path.join(dirpath, baseName + fromSuffix)
  217. fromPxd = os.path.join(dirpath, baseName + ".pxd.in")
  218. toDir = os.path.join(patchDir, subpath)
  219. toPyx = os.path.join(toDir, baseName + ".pyx")
  220. toPxd = os.path.join(toDir, baseName + ".pxd")
  221. # Construct the new cython module name
  222. cyModName = [ "%s_cython" % sourceModName ]
  223. if subpath:
  224. cyModName.extend(subpath.split(os.sep))
  225. cyModName.append(baseName)
  226. cyModName = ".".join(cyModName)
  227. # Remember the filenames for the build
  228. unit = CythonBuildUnit(cyModName, baseName, fromPy, fromPxd,
  229. toDir, toPyx, toPxd)
  230. _cythonBuildUnits.append(unit)
  231. if baseName != "__init__":
  232. # Create a distutils Extension for the module
  233. ext_modules.append(
  234. _Cython_Distutils_Extension(
  235. cyModName,
  236. [toPyx],
  237. cython_directives={
  238. "profile" : profileEnabled,
  239. }
  240. )
  241. )
  242. def registerCythonModules():
  243. baseDir = os.curdir # Base directory, where setup.py lives.
  244. for filename in os.listdir(baseDir):
  245. if os.path.isdir(os.path.join(baseDir, filename)) and\
  246. os.path.exists(os.path.join(baseDir, filename, "__init__.py")) and\
  247. not os.path.exists(os.path.join(baseDir, filename, "no_cython")):
  248. registerCythonModule(baseDir, filename)
  249. def cythonBuildPossible():
  250. global _cythonPossible
  251. if _cythonPossible is not None:
  252. return _cythonPossible
  253. _cythonPossible = False
  254. if os.name != "posix":
  255. print("WARNING: Not building CYTHON modules on '%s' platform." %\
  256. os.name)
  257. return False
  258. if "bdist_wininst" in sys.argv:
  259. print("WARNING: Omitting CYTHON modules while building "
  260. "Windows installer.")
  261. return False
  262. try:
  263. import Cython.Compiler.Options
  264. # Omit docstrings in cythoned modules.
  265. Cython.Compiler.Options.docstrings = False
  266. # Generate module exit cleanup code.
  267. Cython.Compiler.Options.generate_cleanup_code = True
  268. # Generate HTML outputs.
  269. Cython.Compiler.Options.annotate = True
  270. from Cython.Distutils import build_ext, Extension
  271. global _Cython_Distutils_build_ext
  272. global _Cython_Distutils_Extension
  273. _Cython_Distutils_build_ext = build_ext
  274. _Cython_Distutils_Extension = Extension
  275. except ImportError as e:
  276. print("WARNING: Could not build the CYTHON modules: "
  277. "%s" % str(e))
  278. print("--> Is Cython installed?")
  279. return False
  280. _cythonPossible = True
  281. return True
  282. if sys.version_info[0] < 3:
  283. # Cython2 build libraries need method pickling
  284. # for parallel build.
  285. def unpickle_method(fname, obj, cls):
  286. # Ignore MRO. We don't seem to inherit methods.
  287. return cls.__dict__[fname].__get__(obj, cls)
  288. def pickle_method(m):
  289. return unpickle_method, (m.im_func.__name__,
  290. m.im_self,
  291. m.im_class)
  292. import copy_reg, types
  293. copy_reg.pickle(types.MethodType, pickle_method, unpickle_method)
  294. def cyBuildWrapper(arg):
  295. # This function does the same thing as the for-loop-body
  296. # inside of Cython's build_ext.build_extensions() method.
  297. # It is called via multiprocessing to build extensions
  298. # in parallel.
  299. # Note that this might break, if Cython's build_extensions()
  300. # is changed and stuff is added to its for loop. Meh.
  301. self, ext = arg
  302. ext.sources = self.cython_sources(ext.sources, ext)
  303. self.build_extension(ext)
  304. if cythonBuildPossible():
  305. # Override Cython's build_ext class.
  306. class CythonBuildExtension(_Cython_Distutils_build_ext):
  307. def build_extension(self, ext):
  308. assert(not ext.name.endswith("__init__"))
  309. _Cython_Distutils_build_ext.build_extension(self, ext)
  310. def build_extensions(self):
  311. global parallelBuild
  312. # First patch the files, the run the build
  313. patchCythonModules(self.build_lib)
  314. if parallelBuild:
  315. # Run the parallel build, yay.
  316. try:
  317. self.check_extensions_list(self.extensions)
  318. from multiprocessing.pool import Pool
  319. Pool().map(cyBuildWrapper,
  320. ((self, ext) for ext in self.extensions))
  321. except OSError as e:
  322. # This might happen in a restricted
  323. # environment like chroot.
  324. print("WARNING: Parallel build "
  325. "disabled due to: %s" % str(e))
  326. parallelBuild = False
  327. if not parallelBuild:
  328. # Run the normal non-parallel build.
  329. _Cython_Distutils_build_ext.build_extensions(self)