kochdocs.nim 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385
  1. ## Part of 'koch' responsible for the documentation generation.
  2. import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs]
  3. import officialpackages
  4. export exec
  5. when defined(nimPreviewSlimSystem):
  6. import std/assertions
  7. from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry
  8. import "../compiler/nimpaths"
  9. const
  10. gaCode* = " --doc.googleAnalytics:UA-48159761-1"
  11. paCode* = " --doc.plausibleAnalytics:nim-lang.org"
  12. # errormax: subsequent errors are probably consequences of 1st one; a simple
  13. # bug could cause unlimited number of errors otherwise, hard to debug in CI.
  14. docDefines = "-d:nimExperimentalLinenoiseExtra"
  15. nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines]
  16. gitUrl = "https://github.com/nim-lang/Nim"
  17. docHtmlOutput = "doc/html"
  18. webUploadOutput = "web/upload"
  19. var nimExe*: string
  20. const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"]
  21. template isJsOnly(file: string): bool =
  22. file.isRelativeTo("lib/js") or
  23. file.extractFilename in allowList
  24. proc exe*(f: string): string =
  25. result = addFileExt(f, ExeExt)
  26. when defined(windows):
  27. result = result.replace('/','\\')
  28. proc findNimImpl*(): tuple[path: string, ok: bool] =
  29. if nimExe.len > 0: return (nimExe, true)
  30. let nim = "nim".exe
  31. result.path = "bin" / nim
  32. result.ok = true
  33. if fileExists(result.path): return
  34. for dir in split(getEnv("PATH"), PathSep):
  35. result.path = dir / nim
  36. if fileExists(result.path): return
  37. # assume there is a symlink to the exe or something:
  38. return (nim, false)
  39. proc findNim*(): string = findNimImpl().path
  40. template inFold*(desc, body) =
  41. if existsEnv("GITHUB_ACTIONS"):
  42. echo "::group::" & desc
  43. elif existsEnv("TF_BUILD"):
  44. echo "##[group]" & desc
  45. body
  46. if existsEnv("GITHUB_ACTIONS"):
  47. echo "::endgroup::"
  48. elif existsEnv("TF_BUILD"):
  49. echo "##[endgroup]"
  50. proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") =
  51. ## Execute shell command. Add log folding for various CI services.
  52. let desc = if desc.len == 0: cmd else: desc
  53. inFold(desc):
  54. exec(cmd, errorcode, additionalPath)
  55. proc execCleanPath*(cmd: string,
  56. additionalPath = ""; errorcode: int = QuitFailure) =
  57. # simulate a poor man's virtual environment
  58. let prevPath = getEnv("PATH")
  59. when defined(windows):
  60. let cleanPath = r"$1\system32;$1;$1\System32\Wbem" % getEnv"SYSTEMROOT"
  61. else:
  62. const cleanPath = r"/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin"
  63. putEnv("PATH", cleanPath & PathSep & additionalPath)
  64. echo(cmd)
  65. if execShellCmd(cmd) != 0: quit("FAILURE", errorcode)
  66. putEnv("PATH", prevPath)
  67. proc nimexec*(cmd: string) =
  68. # Consider using `nimCompile` instead
  69. exec findNim().quoteShell() & " " & cmd
  70. proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") =
  71. let output = outputDir / input.splitFile.name.exe
  72. let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input
  73. exec cmd
  74. proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") =
  75. let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe
  76. let output = outputDir / outputName2
  77. let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input
  78. execFold(desc, cmd)
  79. const officialPackagesMarkdown = """
  80. pkgs/atlas/doc/atlas.md
  81. """.splitWhitespace()
  82. proc getMd2html(): seq[string] =
  83. for a in walkDirRecFilter("doc"):
  84. let path = a.path
  85. if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin
  86. ["docs.md",
  87. "docstyle.md" # docstyle.md shouldn't be converted to html separately;
  88. # it's included in contributing.md.
  89. ]:
  90. # `docs` is redundant with `overview`, might as well remove that file?
  91. result.add path
  92. for md in officialPackagesMarkdown:
  93. result.add md
  94. doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check
  95. const
  96. mdPdfList = """
  97. manual.md
  98. lib.md
  99. tut1.md
  100. tut2.md
  101. tut3.md
  102. nimc.md
  103. niminst.md
  104. mm.md
  105. """.splitWhitespace().mapIt("doc" / it)
  106. withoutIndex = """
  107. lib/wrappers/tinyc.nim
  108. lib/wrappers/pcre.nim
  109. lib/wrappers/openssl.nim
  110. lib/posix/posix.nim
  111. lib/posix/linux.nim
  112. lib/posix/termios.nim
  113. """.splitWhitespace()
  114. # some of these are include files so shouldn't be docgen'd
  115. ignoredModules = """
  116. lib/pure/future.nim
  117. lib/pure/collections/hashcommon.nim
  118. lib/pure/collections/tableimpl.nim
  119. lib/pure/collections/setimpl.nim
  120. lib/pure/ioselects/ioselectors_kqueue.nim
  121. lib/pure/ioselects/ioselectors_select.nim
  122. lib/pure/ioselects/ioselectors_poll.nim
  123. lib/pure/ioselects/ioselectors_epoll.nim
  124. lib/posix/posix_macos_amd64.nim
  125. lib/posix/posix_other.nim
  126. lib/posix/posix_nintendoswitch.nim
  127. lib/posix/posix_nintendoswitch_consts.nim
  128. lib/posix/posix_linux_amd64.nim
  129. lib/posix/posix_linux_amd64_consts.nim
  130. lib/posix/posix_other_consts.nim
  131. lib/posix/posix_freertos_consts.nim
  132. lib/posix/posix_openbsd_amd64.nim
  133. lib/posix/posix_haiku.nim
  134. lib/pure/md5.nim
  135. lib/std/sha1.nim
  136. """.splitWhitespace()
  137. officialPackagesList = """
  138. pkgs/asyncftpclient/src/asyncftpclient.nim
  139. pkgs/smtp/src/smtp.nim
  140. pkgs/punycode/src/punycode.nim
  141. pkgs/db_connector/src/db_connector/db_common.nim
  142. pkgs/db_connector/src/db_connector/db_mysql.nim
  143. pkgs/db_connector/src/db_connector/db_odbc.nim
  144. pkgs/db_connector/src/db_connector/db_postgres.nim
  145. pkgs/db_connector/src/db_connector/db_sqlite.nim
  146. pkgs/checksums/src/checksums/md5.nim
  147. pkgs/checksums/src/checksums/sha1.nim
  148. pkgs/checksums/src/checksums/sha2.nim
  149. pkgs/checksums/src/checksums/sha3.nim
  150. pkgs/checksums/src/checksums/bcrypt.nim
  151. """.splitWhitespace()
  152. officialPackagesListWithoutIndex = """
  153. pkgs/db_connector/src/db_connector/mysql.nim
  154. pkgs/db_connector/src/db_connector/sqlite3.nim
  155. pkgs/db_connector/src/db_connector/postgres.nim
  156. pkgs/db_connector/src/db_connector/odbcsql.nim
  157. pkgs/db_connector/src/db_connector/private/dbutils.nim
  158. """.splitWhitespace()
  159. when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
  160. proc isRelativeTo(path, base: string): bool =
  161. let path = path.normalizedPath
  162. let base = base.normalizedPath
  163. let ret = relativePath(path, base)
  164. result = path.len > 0 and not ret.startsWith ".."
  165. proc getDocList(): seq[string] =
  166. var docIgnore: HashSet[string]
  167. for a in withoutIndex: docIgnore.incl a
  168. for a in ignoredModules: docIgnore.incl a
  169. # don't ignore these even though in lib/system (not include files)
  170. const goodSystem = """
  171. lib/system/nimscript.nim
  172. lib/system/assertions.nim
  173. lib/system/iterators.nim
  174. lib/system/exceptions.nim
  175. lib/system/dollars.nim
  176. lib/system/ctypes.nim
  177. """.splitWhitespace()
  178. proc follow(a: PathEntry): bool =
  179. result = a.path.lastPathPart notin ["nimcache", htmldocsDirname,
  180. "includes", "deprecated", "genode"] and
  181. not a.path.isRelativeTo("lib/fusion") # fusion was un-bundled but we need to keep this in case user has it installed
  182. for entry in walkDirRecFilter("lib", follow = follow):
  183. let a = entry.path
  184. if entry.kind != pcFile or a.splitFile.ext != ".nim" or
  185. (a.isRelativeTo("lib/system") and a.nativeToUnixPath notin goodSystem) or
  186. a.nativeToUnixPath in docIgnore:
  187. continue
  188. result.add a
  189. result.add normalizePath("nimsuggest/sexp.nim")
  190. let doc = getDocList()
  191. proc sexec(cmds: openArray[string]) =
  192. ## Serial queue wrapper around exec.
  193. for cmd in cmds:
  194. echo(cmd)
  195. let (outp, exitCode) = osproc.execCmdEx(cmd)
  196. if exitCode != 0: quit outp
  197. proc mexec(cmds: openArray[string]) =
  198. ## Multiprocessor version of exec
  199. let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd})
  200. if r != 0:
  201. echo "external program failed, retrying serial work queue for logs!"
  202. sexec(cmds)
  203. proc buildDocSamples(nimArgs, destPath: string) =
  204. ## Special case documentation sample proc.
  205. ##
  206. ## TODO: consider integrating into the existing generic documentation builders
  207. ## now that we have a single `doc` command.
  208. exec(findNim().quoteShell() & " doc $# -o:$# $#" %
  209. [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"])
  210. proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) =
  211. # compiler docs; later, other packages (perhaps tools, testament etc)
  212. let nim = findNim().quoteShell()
  213. # to avoid broken links to manual from compiler dir, but a multi-package
  214. # structure could be supported later
  215. proc docProject(outdir, options, mainproj: string) =
  216. exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [
  217. "nim", nim,
  218. "outdir", outdir,
  219. "nimArgs", nimArgs,
  220. "gitUrl", gitUrl,
  221. "options", options,
  222. "mainproj", mainproj,
  223. "index", if indexOnly: "--index:only" else: ""
  224. ])
  225. let extra = "-u:boot"
  226. # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather,
  227. # start using nims instead of nimdoc.cfg
  228. docProject(destPath/"compiler", extra, "compiler/index.nim")
  229. proc buildDoc(nimArgs, destPath: string, indexOnly: bool) =
  230. # call nim for the documentation:
  231. let rst2html = getMd2html()
  232. var
  233. commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len +
  234. officialPackagesList.len + officialPackagesListWithoutIndex.len)
  235. i = 0
  236. let nim = findNim().quoteShell()
  237. let index = if indexOnly: "--index:only" else: ""
  238. for d in items(rst2html):
  239. commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" %
  240. [nimArgs, gitUrl,
  241. destPath / changeFileExt(splitFile(d).name, "html"), index, d]
  242. i.inc
  243. for d in items(doc):
  244. let extra = if isJsOnly(d): "--backend:js" else: ""
  245. var nimArgs2 = nimArgs
  246. if d.isRelativeTo("compiler"): doAssert false
  247. commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" %
  248. [extra, nimArgs2, gitUrl, destPath, index, d]
  249. i.inc
  250. for d in items(withoutIndex):
  251. commands[i] = nim & " doc $# --git.url:$# -o:$# $#" %
  252. [nimArgs, gitUrl,
  253. destPath / changeFileExt(splitFile(d).name, "html"), d]
  254. i.inc
  255. for d in items(officialPackagesList):
  256. var nimArgs2 = nimArgs
  257. if d.isRelativeTo("compiler"): doAssert false
  258. commands[i] = nim & " doc $# --outdir:$# --index:on $#" %
  259. [nimArgs2, destPath, d]
  260. i.inc
  261. for d in items(officialPackagesListWithoutIndex):
  262. commands[i] = nim & " doc $# -o:$# $#" %
  263. [nimArgs,
  264. destPath / changeFileExt(splitFile(d).name, "html"), d]
  265. i.inc
  266. mexec(commands)
  267. proc nim2pdf(src: string, dst: string, nimArgs: string) =
  268. # xxx expose as a `nim` command or in some other reusable way.
  269. let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616
  270. # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex
  271. exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell])
  272. let texFile = outDir / src.lastPathPart.changeFileExt("tex")
  273. for i in 0..<3: # call LaTeX three times to get cross references right:
  274. let xelatexLog = outDir / "xelatex.log"
  275. # `>` should work on windows, if not, we can use `execCmdEx`
  276. let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell]
  277. exec(cmd) # on error, user can inspect `xelatexLog`
  278. if i == 1: # build .ind file
  279. var texFileBase = texFile
  280. texFileBase.removeSuffix(".tex")
  281. let cmd = "makeindex $# > $#" % [
  282. texFileBase.quoteShell, xelatexLog.quoteShell]
  283. exec(cmd)
  284. moveFile(texFile.changeFileExt("pdf"), dst)
  285. proc buildPdfDoc*(nimArgs, destPath: string) =
  286. var pdfList: seq[string]
  287. createDir(destPath)
  288. if os.execShellCmd("xelatex -version") != 0:
  289. doAssert false, "xelatex not found" # or, raise an exception
  290. else:
  291. for src in items(mdPdfList):
  292. let dst = destPath / src.lastPathPart.changeFileExt("pdf")
  293. pdfList.add dst
  294. nim2pdf(src, dst, nimArgs)
  295. echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose
  296. proc buildJS(): string =
  297. let nim = findNim()
  298. exec("$# js -d:release --out:$# tools/nimblepkglist.nim" %
  299. [nim.quoteShell(), webUploadOutput / "nimblepkglist.js"])
  300. # xxx deadcode? and why is it only for webUploadOutput, not for local docs?
  301. result = getDocHacksJs(nimr = getCurrentDir(), nim)
  302. proc buildDocsDir*(args: string, dir: string) =
  303. let args = nimArgs & " " & args
  304. let docHackJsSource = buildJS()
  305. gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas"])
  306. createDir(dir)
  307. buildDocSamples(args, dir)
  308. # generate `.idx` files and top-level `theindex.html`:
  309. buildDoc(args, dir, indexOnly=true) # bottleneck
  310. let nim = findNim().quoteShell()
  311. exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir])
  312. # caveat: this works so long it's called before `buildDocPackages` which
  313. # populates `compiler/` with unrelated idx files that shouldn't be in index,
  314. # so should work in CI but you may need to remove your generated html files
  315. # locally after calling `./koch docs`. The clean fix would be for `idx` files
  316. # to be transient with `--project` (eg all in memory).
  317. buildDocPackages(args, dir, indexOnly=true)
  318. # generate HTML and package-level `theindex.html`:
  319. buildDoc(args, dir, indexOnly=false) # bottleneck
  320. buildDocPackages(args, dir, indexOnly=false)
  321. copyFile(dir / "overview.html", dir / "index.html")
  322. copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart)
  323. proc buildDocs*(args: string, localOnly = false, localOutDir = "") =
  324. let localOutDir =
  325. if localOutDir.len == 0:
  326. docHtmlOutput
  327. else:
  328. localOutDir
  329. var args = args
  330. if not localOnly:
  331. buildDocsDir(args, webUploadOutput / NimVersion)
  332. let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )"
  333. args = args.replace(gaFilter)
  334. buildDocsDir(args, localOutDir)