kochdocs.nim 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. ## Part of 'koch' responsible for the documentation generation.
  2. import std/[os, strutils, osproc, sets, pathnorm, sequtils]
  3. import officialpackages
  4. export exec
  5. when defined(nimPreviewSlimSystem):
  6. import std/assertions
  7. # XXX: Remove this feature check once the csources supports it.
  8. when defined(nimHasCastPragmaBlocks):
  9. import std/pegs
  10. from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry
  11. import "../compiler/nimpaths"
  12. const
  13. gaCode* = " --doc.googleAnalytics:UA-48159761-1"
  14. paCode* = " --doc.plausibleAnalytics:nim-lang.org"
  15. # errormax: subsequent errors are probably consequences of 1st one; a simple
  16. # bug could cause unlimited number of errors otherwise, hard to debug in CI.
  17. docDefines = "-d:nimExperimentalLinenoiseExtra"
  18. nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines]
  19. gitUrl = "https://github.com/nim-lang/Nim"
  20. docHtmlOutput = "doc/html"
  21. webUploadOutput = "web/upload"
  22. var nimExe*: string
  23. const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"]
  24. template isJsOnly(file: string): bool =
  25. file.isRelativeTo("lib/js") or
  26. file.extractFilename in allowList
  27. proc exe*(f: string): string =
  28. result = addFileExt(f, ExeExt)
  29. when defined(windows):
  30. result = result.replace('/','\\')
  31. proc findNimImpl*(): tuple[path: string, ok: bool] =
  32. if nimExe.len > 0: return (nimExe, true)
  33. let nim = "nim".exe
  34. result.path = "bin" / nim
  35. result.ok = true
  36. if fileExists(result.path): return
  37. for dir in split(getEnv("PATH"), PathSep):
  38. result.path = dir / nim
  39. if fileExists(result.path): return
  40. # assume there is a symlink to the exe or something:
  41. return (nim, false)
  42. proc findNim*(): string = findNimImpl().path
  43. template inFold*(desc, body) =
  44. if existsEnv("GITHUB_ACTIONS"):
  45. echo "::group::" & desc
  46. elif existsEnv("TF_BUILD"):
  47. echo "##[group]" & desc
  48. body
  49. if existsEnv("GITHUB_ACTIONS"):
  50. echo "::endgroup::"
  51. elif existsEnv("TF_BUILD"):
  52. echo "##[endgroup]"
  53. proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
  54. ## Execute shell command. Add log folding for various CI services.
  55. let desc = if desc.len == 0: cmd else: desc
  56. inFold(desc):
  57. exec(cmd, errorcode, additionalPath)
  58. proc execCleanPath*(cmd: string,
  59. additionalPath = ""; errorcode: int = QuitFailure) =
  60. # simulate a poor man's virtual environment
  61. let prevPath = getEnv("PATH")
  62. when defined(windows):
  63. let cleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT"
  64. else:
  65. const cleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin"
  66. putEnv("PATH", cleanPath & PathSep & additionalPath)
  67. echo(cmd)
  68. if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
  69. putEnv("PATH", prevPath)
  70. proc nimexec*(cmd: string) =
  71. # Consider using `nimCompile` instead
  72. exec findNim().quoteShell() & " " & cmd
  73. proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") =
  74. let output = outputDir / input.splitFile.name.exe
  75. let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input
  76. exec cmd
  77. proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") =
  78. let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe
  79. let output = outputDir / outputName2
  80. let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input
  81. execFold(desc, cmd)
  82. proc getMd2html(): seq[string] =
  83. for a in walkDirRecFilter("doc"):
  84. let path = a.path
  85. if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin
  86. ["docs.md", "nimfix.md",
  87. "docstyle.md" # docstyle.md shouldn't be converted to html separately;
  88. # it's included in contributing.md.
  89. ]:
  90. # maybe we should still show nimfix, could help reviving it
  91. # `docs` is redundant with `overview`, might as well remove that file?
  92. result.add path
  93. doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check
  94. const
  95. mdPdfList = """
  96. manual.md
  97. lib.md
  98. tut1.md
  99. tut2.md
  100. tut3.md
  101. nimc.md
  102. niminst.md
  103. mm.md
  104. """.splitWhitespace().mapIt("doc" / it)
  105. withoutIndex = """
  106. lib/wrappers/tinyc.nim
  107. lib/wrappers/pcre.nim
  108. lib/wrappers/openssl.nim
  109. lib/posix/posix.nim
  110. lib/posix/linux.nim
  111. lib/posix/termios.nim
  112. """.splitWhitespace()
  113. # some of these are include files so shouldn't be docgen'd
  114. ignoredModules = """
  115. lib/pure/future.nim
  116. lib/pure/collections/hashcommon.nim
  117. lib/pure/collections/tableimpl.nim
  118. lib/pure/collections/setimpl.nim
  119. lib/pure/ioselects/ioselectors_kqueue.nim
  120. lib/pure/ioselects/ioselectors_select.nim
  121. lib/pure/ioselects/ioselectors_poll.nim
  122. lib/pure/ioselects/ioselectors_epoll.nim
  123. lib/posix/posix_macos_amd64.nim
  124. lib/posix/posix_other.nim
  125. lib/posix/posix_nintendoswitch.nim
  126. lib/posix/posix_nintendoswitch_consts.nim
  127. lib/posix/posix_linux_amd64.nim
  128. lib/posix/posix_linux_amd64_consts.nim
  129. lib/posix/posix_other_consts.nim
  130. lib/posix/posix_freertos_consts.nim
  131. lib/posix/posix_openbsd_amd64.nim
  132. lib/posix/posix_haiku.nim
  133. """.splitWhitespace()
  134. officialPackagesList = """
  135. pkgs/asyncftpclient/src/asyncftpclient.nim
  136. pkgs/smtp/src/smtp.nim
  137. pkgs/punycode/src/punycode.nim
  138. pkgs/db_connector/src/db_connector/db_common.nim
  139. pkgs/db_connector/src/db_connector/db_mysql.nim
  140. pkgs/db_connector/src/db_connector/db_odbc.nim
  141. pkgs/db_connector/src/db_connector/db_postgres.nim
  142. pkgs/db_connector/src/db_connector/db_sqlite.nim
  143. """.splitWhitespace()
  144. officialPackagesListWithoutIndex = """
  145. pkgs/db_connector/src/db_connector/mysql.nim
  146. pkgs/db_connector/src/db_connector/sqlite3.nim
  147. pkgs/db_connector/src/db_connector/postgres.nim
  148. pkgs/db_connector/src/db_connector/odbcsql.nim
  149. pkgs/db_connector/src/db_connector/private/dbutils.nim
  150. """.splitWhitespace()
  151. proc findName(name: string): string =
  152. doAssert name[0..4] == "pkgs/"
  153. var i = 5
  154. while i < name.len:
  155. if name[i] != '/':
  156. inc i
  157. result.add name[i]
  158. else:
  159. break
  160. when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
  161. proc isRelativeTo(path, base: string): bool =
  162. let path = path.normalizedPath
  163. let base = base.normalizedPath
  164. let ret = relativePath(path, base)
  165. result = path.len > 0 and not ret.startsWith ".."
  166. proc getDocList(): seq[string] =
  167. var docIgnore: HashSet[string]
  168. for a in withoutIndex: docIgnore.incl a
  169. for a in ignoredModules: docIgnore.incl a
  170. # don't ignore these even though in lib/system (not include files)
  171. const goodSystem = """
  172. lib/system/nimscript.nim
  173. lib/system/assertions.nim
  174. lib/system/iterators.nim
  175. lib/system/exceptions.nim
  176. lib/system/dollars.nim
  177. lib/system/ctypes.nim
  178. """.splitWhitespace()
  179. proc follow(a: PathEntry): bool =
  180. result = a.path.lastPathPart notin ["nimcache", htmldocsDirname,
  181. "includes", "deprecated", "genode"] and
  182. not a.path.isRelativeTo("lib/fusion") # fusion was un-bundled but we need to keep this in case user has it installed
  183. for entry in walkDirRecFilter("lib", follow = follow):
  184. let a = entry.path
  185. if entry.kind != pcFile or a.splitFile.ext != ".nim" or
  186. (a.isRelativeTo("lib/system") and a.nativeToUnixPath notin goodSystem) or
  187. a.nativeToUnixPath in docIgnore:
  188. continue
  189. result.add a
  190. result.add normalizePath("nimsuggest/sexp.nim")
  191. let doc = getDocList()
  192. proc sexec(cmds: openArray[string]) =
  193. ## Serial queue wrapper around exec.
  194. for cmd in cmds:
  195. echo(cmd)
  196. let (outp, exitCode) = osproc.execCmdEx(cmd)
  197. if exitCode != 0: quit outp
  198. proc mexec(cmds: openArray[string]) =
  199. ## Multiprocessor version of exec
  200. let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd})
  201. if r != 0:
  202. echo "external program failed, retrying serial work queue for logs!"
  203. sexec(cmds)
  204. proc buildDocSamples(nimArgs, destPath: string) =
  205. ## Special case documentation sample proc.
  206. ##
  207. ## TODO: consider integrating into the existing generic documentation builders
  208. ## now that we have a single `doc` command.
  209. exec(findNim().quoteShell() & " doc $# -o:$# $#" %
  210. [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"])
  211. proc buildDocPackages(nimArgs, destPath: string) =
  212. # compiler docs; later, other packages (perhaps tools, testament etc)
  213. let nim = findNim().quoteShell()
  214. # to avoid broken links to manual from compiler dir, but a multi-package
  215. # structure could be supported later
  216. proc docProject(outdir, options, mainproj: string) =
  217. exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [
  218. "nim", nim,
  219. "outdir", outdir,
  220. "nimArgs", nimArgs,
  221. "gitUrl", gitUrl,
  222. "options", options,
  223. "mainproj", mainproj,
  224. ])
  225. let extra = "-u:boot"
  226. # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather,
  227. # start using nims instead of nimdoc.cfg
  228. docProject(destPath/"compiler", extra, "compiler/index.nim")
  229. proc buildDoc(nimArgs, destPath: string) =
  230. # call nim for the documentation:
  231. let rst2html = getMd2html()
  232. var
  233. commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len +
  234. officialPackagesList.len + officialPackagesListWithoutIndex.len)
  235. i = 0
  236. let nim = findNim().quoteShell()
  237. for d in items(rst2html):
  238. commands[i] = nim & " md2html $# --git.url:$# -o:$# --index:on $#" %
  239. [nimArgs, gitUrl,
  240. destPath / changeFileExt(splitFile(d).name, "html"), d]
  241. i.inc
  242. for d in items(doc):
  243. let extra = if isJsOnly(d): "--backend:js" else: ""
  244. var nimArgs2 = nimArgs
  245. if d.isRelativeTo("compiler"): doAssert false
  246. commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" %
  247. [extra, nimArgs2, gitUrl, destPath, d]
  248. i.inc
  249. for d in items(withoutIndex):
  250. commands[i] = nim & " doc $# --git.url:$# -o:$# $#" %
  251. [nimArgs, gitUrl,
  252. destPath / changeFileExt(splitFile(d).name, "html"), d]
  253. i.inc
  254. for d in items(officialPackagesList):
  255. var nimArgs2 = nimArgs
  256. if d.isRelativeTo("compiler"): doAssert false
  257. commands[i] = nim & " doc $# --outdir:$# --index:on $#" %
  258. [nimArgs2, destPath, d]
  259. i.inc
  260. for d in items(officialPackagesListWithoutIndex):
  261. commands[i] = nim & " doc $# -o:$# $#" %
  262. [nimArgs,
  263. destPath / changeFileExt(splitFile(d).name, "html"), d]
  264. i.inc
  265. mexec(commands)
  266. exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath])
  267. # caveat: this works so long it's called before `buildDocPackages` which
  268. # populates `compiler/` with unrelated idx files that shouldn't be in index,
  269. # so should work in CI but you may need to remove your generated html files
  270. # locally after calling `./koch docs`. The clean fix would be for `idx` files
  271. # to be transient with `--project` (eg all in memory).
  272. proc nim2pdf(src: string, dst: string, nimArgs: string) =
  273. # xxx expose as a `nim` command or in some other reusable way.
  274. let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616
  275. # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex
  276. exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell])
  277. let texFile = outDir / src.lastPathPart.changeFileExt("tex")
  278. for i in 0..<3: # call LaTeX three times to get cross references right:
  279. let xelatexLog = outDir / "xelatex.log"
  280. # `>` should work on windows, if not, we can use `execCmdEx`
  281. let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell]
  282. exec(cmd) # on error, user can inspect `xelatexLog`
  283. if i == 1: # build .ind file
  284. var texFileBase = texFile
  285. texFileBase.removeSuffix(".tex")
  286. let cmd = "makeindex $# > $#" % [
  287. texFileBase.quoteShell, xelatexLog.quoteShell]
  288. exec(cmd)
  289. moveFile(texFile.changeFileExt("pdf"), dst)
  290. proc buildPdfDoc*(nimArgs, destPath: string) =
  291. var pdfList: seq[string]
  292. createDir(destPath)
  293. if os.execShellCmd("xelatex -version") != 0:
  294. doAssert false, "xelatex not found" # or, raise an exception
  295. else:
  296. for src in items(mdPdfList):
  297. let dst = destPath / src.lastPathPart.changeFileExt("pdf")
  298. pdfList.add dst
  299. nim2pdf(src, dst, nimArgs)
  300. echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose
  301. proc buildJS(): string =
  302. let nim = findNim()
  303. exec("$# js -d:release --out:$# tools/nimblepkglist.nim" %
  304. [nim.quoteShell(), webUploadOutput / "nimblepkglist.js"])
  305. # xxx deadcode? and why is it only for webUploadOutput, not for local docs?
  306. result = getDocHacksJs(nimr = getCurrentDir(), nim)
  307. proc buildDocsDir*(args: string, dir: string) =
  308. let args = nimArgs & " " & args
  309. let docHackJsSource = buildJS()
  310. gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector"])
  311. createDir(dir)
  312. buildDocSamples(args, dir)
  313. buildDoc(args, dir) # bottleneck
  314. copyFile(dir / "overview.html", dir / "index.html")
  315. buildDocPackages(args, dir)
  316. copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart)
  317. proc buildDocs*(args: string, localOnly = false, localOutDir = "") =
  318. let localOutDir =
  319. if localOutDir.len == 0:
  320. docHtmlOutput
  321. else:
  322. localOutDir
  323. var args = args
  324. if not localOnly:
  325. buildDocsDir(args, webUploadOutput / NimVersion)
  326. # XXX: Remove this feature check once the csources supports it.
  327. when defined(nimHasCastPragmaBlocks):
  328. let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )"
  329. args = args.replace(gaFilter)
  330. buildDocsDir(args, localOutDir)