nimhcr.nim 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672
  1. discard """
  2. batchable: false
  3. """
  4. #
  5. #
  6. # Nim's Runtime Library
  7. # (c) Copyright 2018 Nim Contributors
  8. #
  9. # See the file "copying.txt", included in this
  10. # distribution, for details about the copyright.
  11. #
  12. # This is the Nim hot code reloading run-time for the native targets.
  13. #
  14. # This minimal dynamic library is not subject to reloading when the
  15. # `hotCodeReloading` build mode is enabled. It's responsible for providing
  16. # a permanent memory location for all globals and procs within a program
  17. # and orchestrating the reloading. For globals, this is easily achieved
  18. # by storing them on the heap. For procs, we produce on the fly simple
  19. # trampolines that can be dynamically overwritten to jump to a different
  20. # target. In the host program, all globals and procs are first registered
  21. # here with `hcrRegisterGlobal` and `hcrRegisterProc` and then the
  22. # returned permanent locations are used in every reference to these symbols
  23. # onwards.
  24. #
  25. # Detailed description:
  26. #
  27. # When code is compiled with the hotCodeReloading option for native targets
  28. # a couple of things happen for all modules in a project:
  29. # - the useNimRtl option is forced (including when building the HCR runtime too)
  30. # - all modules of a target get built into separate shared libraries
  31. # - the smallest granularity of reloads is modules
  32. # - for each .c (or .cpp) in the corresponding nimcache folder of the project
  33. # a shared object is built with the name of the source file + DLL extension
  34. # - only the main module produces whatever the original project type intends
  35. # (again in nimcache) and is then copied to its original destination
  36. # - linking is done in parallel - just like compilation
  37. # - function calls to functions from the same project go through function pointers:
  38. # - with a few exceptions - see the nonReloadable pragma
  39. # - the forward declarations of the original functions become function
  40. # pointers as static globals with the same names
  41. # - the original function definitions get suffixed with <name>_actual
  42. # - the function pointers get initialized with the address of the corresponding
  43. # function in the DatInit of their module through a call to either hcrRegisterProc
  44. # or hcrGetProc. When being registered, the <name>_actual address is passed to
  45. # hcrRegisterProc and a permanent location is returned and assigned to the pointer.
  46. # This way the implementation (<name>_actual) can change but the address for it
  47. # will be the same - this works by just updating a jump instruction (trampoline).
  48. # For functions from other modules hcrGetProc is used (after they are registered).
  49. # - globals are initialized only once and their state is preserved
  50. # - including locals with the {.global.} pragma
  51. # - their definitions are changed into pointer definitions which are initialized
  52. # in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the
  53. # size of the type that this HCR runtime should allocate) and a bool is returned
  54. # which when true triggers the initialization code for the global (only once).
  55. # Globals from other modules: a global pointer coupled with a hcrGetGlobal call.
  56. # - globals which have already been initialized cannot have their values changed
  57. # by changing their initialization - use a handler or some other mechanism
  58. # - new globals can be introduced when reloading
  59. # - top-level code (global scope) is executed only once - at the first module load
  60. # - the runtime knows every symbol's module owner (globals and procs)
  61. # - both the RTL and HCR shared libraries need to be near the program for execution
  62. # - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS)
  63. # - the main module is responsible for initializing the HCR runtime
  64. # - the main module loads the RTL and HCR shared objects
  65. # - after that a call to hcrInit() is done in the main module which triggers
  66. # the loading of all modules the main one imports, and doing that for the
  67. # dependencies of each module recursively. Basically a DFS traversal.
  68. # - then initialization takes place with several passes over all modules:
  69. # - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc
  70. # - HcrCreateTypeInfos - creates globals which will be referenced in the next pass
  71. # - DatInit - usual dat init + register/get procs and get globals
  72. # - Init - it does the following multiplexed operations:
  73. # - register globals (if already registered - then just retrieve pointer)
  74. # - execute top level scope (only if loaded for the first time)
  75. # - when modules are loaded the originally built shared libraries get copied in
  76. # the same folder and the copies are loaded instead of the original files
  77. # - a module import tree is built in the runtime (and maintained when reloading)
  78. # - hcrPerformCodeReload
  79. # - named `performCodeReload`, requires the hotcodereloading module
  80. # - explicitly called by the user - the current active callstack shouldn't contain
  81. # any functions which are defined in modules that will be reloaded (or crash!).
  82. # The reason is that old dynamic libraries get unloaded.
  83. # Example:
  84. # if A is the main module and it imports B, then only B is reloadable and only
  85. # if when calling hcrPerformCodeReload there is no function defined in B in the
  86. # current active callstack at the point of the call (it has to be done from A)
  87. # - for reloading to take place the user has to have rebuilt parts of the application
  88. # without changes affecting the main module in any way - it shouldn't be rebuilt.
  89. # - to determine what needs to be reloaded the runtime starts traversing the import
  90. # tree from the root and checks the timestamps of the loaded shared objects
  91. # - modules that are no longer referenced are unloaded and cleaned up properly
  92. # - symbols (procs/globals) that have been removed in the code are also cleaned up
  93. # - so changing the init of a global does nothing, but removing it, reloading,
  94. # and then re-introducing it with a new initializer works
  95. # - new modules can be imported, and imports can also be reodereded/removed
  96. # - hcrReloadNeeded() can be used to determine if any module needs reloading
  97. # - named `hasAnyModuleChanged`, requires the hotcodereloading module
  98. # - code in the beforeCodeReload/afterCodeReload handlers is executed on each reload
  99. # - require the hotcodereloading module
  100. # - such handlers can be added and removed
  101. # - before each reload all "beforeCodeReload" handlers are executed and after
  102. # that all handlers (including "after") from the particular module are deleted
  103. # - the order of execution is the same as the order of top-level code execution.
  104. # Example: if A imports B which imports C, then all handlers in C will be executed
  105. # first (from top to bottom) followed by all from B and lastly all from A
  106. # - after the reload all "after" handlers are executed the same way as "before"
  107. # - the handlers for a reloaded module are always removed when reloading and then
  108. # registered when the top-level scope is executed (thanks to `executeOnReload`)
  109. #
  110. # TODO next:
  111. #
  112. # - implement the before/after handlers and hasModuleChanged for the javascript target
  113. # - ARM support for the trampolines
  114. # - investigate:
  115. # - soon the system module might be importing other modules - the init order...?
  116. # (revert https://github.com/nim-lang/Nim/pull/11971 when working on this)
  117. # - rethink the closure iterators
  118. # - ability to keep old versions of dynamic libraries alive
  119. # - because of async server code
  120. # - perhaps with refcounting of .dlls for unfinished closures
  121. # - linking with static libs
  122. # - all shared objects for each module will (probably) have to link to them
  123. # - state in static libs gets duplicated
  124. # - linking is slow and therefore iteration time suffers
  125. # - have just a single .dll for all .nim files and bulk reload?
  126. # - think about the compile/link/passc/passl/emit/injectStmt pragmas
  127. # - if a passc pragma is introduced (either written or dragged in by a new
  128. # import) the whole command line for compilation changes - for example:
  129. # winlean.nim: {.passc: "-DWIN32_LEAN_AND_MEAN".}
  130. # - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*'
  131. # - everything thread-local related
  132. # - tests
  133. # - add a new travis build matrix entry which builds everything with HCR enabled
  134. # - currently building with useNimRtl is problematic - lots of problems...
  135. # - how to supply the nimrtl/nimhcr shared objects to all test binaries...?
  136. # - think about building to C++ instead of only to C - added type safety
  137. # - run tests through valgrind and the sanitizers!
  138. #
  139. # TODO - nice to have cool stuff:
  140. #
  141. # - separate handling of global state for much faster reloading and manipulation
  142. # - imagine sliders in an IDE for tweaking variables
  143. # - perhaps using shared memory
  144. # - multi-dll projects - how everything can be reloaded..?
  145. # - a single HCR instance shared across multiple .dlls
  146. # - instead of having to call hcrPerformCodeReload from a function in each dll
  147. # - which currently renders the main module of each dll not reloadable
  148. # - ability to check with the current callstack if a reload is "legal"
  149. # - if it is in any function which is in a module about to be reloaded ==> error
  150. # - pragma annotations for files - to be excluded from dll shenanigans
  151. # - for such file-global pragmas look at codeReordering or injectStmt
  152. # - how would the initialization order be kept? messy...
  153. # - C code calling stable exportc interface of nim code (for bindings)
  154. # - generate proxy functions with the stable names
  155. # - in a non-reloadable part (the main binary) that call the function pointers
  156. # - parameter passing/forwarding - how? use the same trampoline jumping?
  157. # - extracting the dependencies for these stubs/proxies will be hard...
  158. # - changing memory layout of types - detecting this..?
  159. # - implement with registerType() call to HCR runtime...?
  160. # - and checking if a previously registered type matches
  161. # - issue an error
  162. # - or let the user handle this by transferring the state properly
  163. # - perhaps in the before/afterCodeReload handlers
  164. # - implement executeOnReload for global vars too - not just statements (and document!)
  165. # - cleanup at shutdown - freeing all globals
  166. # - fallback mechanism if the program crashes (the program should detect crashes
  167. # by itself using SEH/signals on Windows/Unix) - should be able to revert to
  168. # previous versions of the .dlls by calling some function from HCR
  169. # - improve runtime performance - possibilities
  170. # - implement a way for multiple .nim files to be bundled into the same dll
  171. # and have all calls within that domain to use the "_actual" versions of
  172. # procs so there are no indirections (or the ability to just bundle everything
  173. # except for a few unreloadable modules into a single mega reloadable dll)
  174. # - try to load the .dlls at specific addresses of memory (close to each other)
  175. # allocated with execution flags - check this: https://github.com/fancycode/MemoryModule
  176. #
  177. # TODO - unimportant:
  178. #
  179. # - have a "bad call" trampoline that all no-longer-present functions are routed to call there
  180. # - so the user gets some error msg if he calls a dangling pointer instead of a crash
  181. # - before/afterCodeReload and hasModuleChanged should be accessible only where appropriate
  182. # - nim_program_result is inaccessible in HCR mode from external C code (see nimbase.h)
  183. # - proper .json build file - but the format is different... multiple link commands...
  184. # - avoid registering globals on each loop when using an iterator in global scope
  185. #
  186. # TODO - REPL:
  187. # - proper way (as proposed by Zahary):
  188. # - parse the input code and put everything in global scope except for
  189. # statements with side effects only - those go in afterCodeReload blocks
  190. # - my very hacky idea: just append to a closure iterator the new statements
  191. # followed by a yield statement. So far I can think of 2 problems:
  192. # - import and some other code cannot be written inside of a proc -
  193. # has to be parsed and extracted in the outer scope
  194. # - when new variables are created they are actually locals to the closure
  195. # so the struct for the closure state grows in memory, but it has already
  196. # been allocated when the closure was created with the previous smaller size.
  197. # That would lead to working with memory outside of the initially allocated
  198. # block. Perhaps something can be done about this - some way of re-allocating
  199. # the state and transferring the old...
  200. when defined(nimPreviewSlimSystem):
  201. import std/assertions
  202. when not defined(js) and (defined(hotcodereloading) or
  203. defined(createNimHcr) or
  204. defined(testNimHcr)):
  205. const
  206. dllExt = when defined(windows): "dll"
  207. elif defined(macosx): "dylib"
  208. else: "so"
  209. type
  210. HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.}
  211. HcrGcMarkerProc = proc () {.nimcall, raises: [].}
  212. HcrModuleInitializer* = proc () {.nimcall.}
  213. when defined(createNimHcr):
  214. when system.appType != "lib":
  215. {.error: "This file has to be compiled as a library!".}
  216. import std/[os, tables, sets, times, strutils, reservedmem, dynlib]
  217. template trace(args: varargs[untyped]) =
  218. when defined(testNimHcr) or defined(traceHcr):
  219. echo args
  220. proc sanitize(arg: Time): string =
  221. when defined(testNimHcr): return "<time>"
  222. else: return $arg
  223. proc sanitize(arg: string|cstring): string =
  224. when defined(testNimHcr): return ($arg).splitFile.name.splitFile.name
  225. else: return $arg
  226. {.pragma: nimhcr, compilerproc, exportc, dynlib.}
  227. # XXX these types are CPU specific and need ARM etc support
  228. type
  229. ShortJumpInstruction {.packed.} = object
  230. opcode: byte
  231. offset: int32
  232. LongJumpInstruction {.packed.} = object
  233. opcode1: byte
  234. opcode2: byte
  235. offset: int32
  236. absoluteAddr: pointer
  237. proc writeJump(jumpTableEntry: ptr LongJumpInstruction, targetFn: pointer) =
  238. let
  239. jumpFrom = jumpTableEntry.shift(sizeof(ShortJumpInstruction))
  240. jumpDistance = distance(jumpFrom, targetFn)
  241. if abs(jumpDistance) < 0x7fff0000:
  242. let shortJump = cast[ptr ShortJumpInstruction](jumpTableEntry)
  243. shortJump.opcode = 0xE9 # relative jump
  244. shortJump.offset = int32(jumpDistance)
  245. else:
  246. jumpTableEntry.opcode1 = 0xff # indirect absolute jump
  247. jumpTableEntry.opcode2 = 0x25
  248. when hostCPU == "i386":
  249. # on x86 we write the absolute address of the following pointer
  250. jumpTableEntry.offset = cast[int32](addr jumpTableEntry.absoluteAddr)
  251. else:
  252. # on x64, we use a relative address for the same location
  253. jumpTableEntry.offset = 0
  254. jumpTableEntry.absoluteAddr = targetFn
  255. if hostCPU == "arm":
  256. const jumpSize = 8
  257. elif hostCPU == "arm64":
  258. const jumpSize = 16
  259. const defaultJumpTableSize = case hostCPU
  260. of "i386": 50
  261. of "amd64": 500
  262. else: 50
  263. let jumpTableSizeStr = getEnv("HOT_CODE_RELOADING_JUMP_TABLE_SIZE")
  264. let jumpTableSize = if jumpTableSizeStr.len > 0: parseInt(jumpTableSizeStr)
  265. else: defaultJumpTableSize
  266. # TODO: perhaps keep track of free slots due to removed procs using a free list
  267. var jumpTable = ReservedMemSeq[LongJumpInstruction].init(
  268. memStart = cast[pointer](0x10000000),
  269. maxLen = jumpTableSize * 1024 * 1024 div sizeof(LongJumpInstruction),
  270. accessFlags = memExecReadWrite)
  271. type
  272. ProcSym = object
  273. jump: ptr LongJumpInstruction
  274. gen: int
  275. GlobalVarSym = object
  276. p: pointer
  277. markerProc: HcrGcMarkerProc
  278. gen: int
  279. ModuleDesc = object
  280. procs: Table[string, ProcSym]
  281. globals: Table[string, GlobalVarSym]
  282. imports: seq[string]
  283. handle: LibHandle
  284. hash: string
  285. gen: int
  286. lastModification: Time
  287. handlers: seq[tuple[isBefore: bool, cb: proc () {.nimcall.}]]
  288. proc newModuleDesc(): ModuleDesc =
  289. result.procs = initTable[string, ProcSym]()
  290. result.globals = initTable[string, GlobalVarSym]()
  291. result.handle = nil
  292. result.gen = -1
  293. result.lastModification = low(Time)
  294. # the global state necessary for traversing and reloading the module import tree
  295. var modules = initTable[string, ModuleDesc]()
  296. var root: string
  297. var system: string
  298. var mainDatInit: HcrModuleInitializer
  299. var generation = 0
  300. # necessary for queries such as "has module X changed" - contains all but the main module
  301. var hashToModuleMap = initTable[string, string]()
  302. # necessary for registering handlers and keeping them up-to-date
  303. var currentModule: string
  304. # supplied from the main module - used by others to initialize pointers to this runtime
  305. var hcrDynlibHandle: pointer
  306. var getProcAddr: HcrProcGetter
  307. proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} =
  308. trace " register proc: ", module.sanitize, " ", name
  309. # Please note: We must allocate a local copy of the strings, because the supplied
  310. # `cstring` will reside in the data segment of a DLL that will be later unloaded.
  311. let name = $name
  312. let module = $module
  313. var jumpTableEntryAddr: ptr LongJumpInstruction
  314. modules[module].procs.withValue(name, p):
  315. trace " update proc: ", name
  316. jumpTableEntryAddr = p.jump
  317. p.gen = generation
  318. do:
  319. let len = jumpTable.len
  320. jumpTable.setLen(len + 1)
  321. jumpTableEntryAddr = addr jumpTable[len]
  322. modules[module].procs[name] = ProcSym(jump: jumpTableEntryAddr, gen: generation)
  323. writeJump jumpTableEntryAddr, fn
  324. return jumpTableEntryAddr
  325. proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} =
  326. trace " get proc: ", module.sanitize, " ", name
  327. return modules[$module].procs.getOrDefault($name, ProcSym()).jump
  328. proc hcrRegisterGlobal*(module: cstring,
  329. name: cstring,
  330. size: Natural,
  331. gcMarker: HcrGcMarkerProc,
  332. outPtr: ptr pointer): bool {.nimhcr.} =
  333. trace " register global: ", module.sanitize, " ", name
  334. # Please note: We must allocate local copies of the strings, because the supplied
  335. # `cstring` will reside in the data segment of a DLL that will be later unloaded.
  336. # Also using a ptr pointer instead of a var pointer (an output parameter)
  337. # because for the C++ backend var parameters use references and in this use case
  338. # it is not possible to cast an int* (for example) to a void* and then pass it
  339. # to void*& since the casting yields an rvalue and references bind only to lvalues.
  340. let name = $name
  341. let module = $module
  342. modules[module].globals.withValue(name, global):
  343. trace " update global: ", name
  344. outPtr[] = global.p
  345. global.gen = generation
  346. global.markerProc = gcMarker
  347. return false
  348. do:
  349. outPtr[] = alloc0(size)
  350. modules[module].globals[name] = GlobalVarSym(p: outPtr[],
  351. gen: generation,
  352. markerProc: gcMarker)
  353. return true
  354. proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} =
  355. trace " get global: ", module.sanitize, " ", name
  356. return modules[$module].globals[$name].p
  357. proc getListOfModules(cstringArray: ptr pointer): seq[string] =
  358. var curr = cast[ptr cstring](cstringArray)
  359. while len(curr[]) > 0:
  360. result.add($curr[])
  361. curr = cast[ptr cstring](cast[int64](curr) + sizeof(ptr cstring))
  362. template cleanup(collection, body) =
  363. var toDelete: seq[string]
  364. for name, data in collection.pairs:
  365. if data.gen < generation:
  366. toDelete.add(name)
  367. trace "HCR Cleaning ", astToStr(collection), " :: ", name, " ", data.gen
  368. for name {.inject.} in toDelete:
  369. body
  370. proc cleanupGlobal(module: string, name: string) =
  371. var g: GlobalVarSym
  372. if modules[module].globals.take(name, g):
  373. dealloc g.p
  374. proc cleanupSymbols(module: string) =
  375. cleanup modules[module].globals:
  376. cleanupGlobal(module, name)
  377. cleanup modules[module].procs:
  378. modules[module].procs.del(name)
  379. proc unloadDll(name: string) =
  380. if modules[name].handle != nil:
  381. unloadLib(modules[name].handle)
  382. proc loadDll(name: cstring) {.nimhcr.} =
  383. let name = $name
  384. trace "HCR LOADING: ", name.sanitize
  385. if modules.contains(name):
  386. unloadDll(name)
  387. else:
  388. modules[name] = newModuleDesc()
  389. let copiedName = name & ".copy." & dllExt
  390. copyFileWithPermissions(name, copiedName)
  391. let lib = loadLib(copiedName)
  392. assert lib != nil
  393. modules[name].handle = lib
  394. modules[name].gen = generation
  395. modules[name].lastModification = getLastModificationTime(name)
  396. # update the list of imports by the module
  397. let getImportsProc = cast[proc (): ptr pointer {.nimcall.}](
  398. checkedSymAddr(lib, "HcrGetImportedModules"))
  399. modules[name].imports = getListOfModules(getImportsProc())
  400. # get the hash of the module
  401. let getHashProc = cast[proc (): cstring {.nimcall.}](
  402. checkedSymAddr(lib, "HcrGetSigHash"))
  403. modules[name].hash = $getHashProc()
  404. hashToModuleMap[modules[name].hash] = name
  405. # Remove handlers for this module if reloading - they will be re-registered.
  406. # In order for them to be re-registered we need to de-register all globals
  407. # that trigger the registering of handlers through calls to hcrAddEventHandler
  408. modules[name].handlers.setLen(0)
  409. proc initHcrData(name: cstring) {.nimhcr.} =
  410. trace "HCR Hcr init: ", name.sanitize
  411. cast[proc (h: pointer, gpa: HcrProcGetter) {.nimcall.}](
  412. checkedSymAddr(modules[$name].handle, "HcrInit000"))(hcrDynlibHandle, getProcAddr)
  413. proc initTypeInfoGlobals(name: cstring) {.nimhcr.} =
  414. trace "HCR TypeInfo globals init: ", name.sanitize
  415. cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "HcrCreateTypeInfos"))()
  416. proc initPointerData(name: cstring) {.nimhcr.} =
  417. trace "HCR Dat init: ", name.sanitize
  418. cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "DatInit000"))()
  419. proc initGlobalScope(name: cstring) {.nimhcr.} =
  420. trace "HCR Init000: ", name.sanitize
  421. # set the currently inited module - necessary for registering the before/after HCR handlers
  422. currentModule = $name
  423. cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "Init000"))()
  424. var modulesToInit: seq[string] = @[]
  425. var allModulesOrderedByDFS: seq[string] = @[]
  426. proc recursiveDiscovery(dlls: seq[string]) =
  427. for curr in dlls:
  428. if modules.contains(curr):
  429. # skip updating modules that have already been updated to the latest generation
  430. if modules[curr].gen >= generation:
  431. trace "HCR SKIP: ", curr.sanitize, " gen is already: ", modules[curr].gen
  432. continue
  433. # skip updating an unmodified module but continue traversing its dependencies
  434. if modules[curr].lastModification >= getLastModificationTime(curr):
  435. trace "HCR SKIP (not modified): ", curr.sanitize, " ", modules[curr].lastModification.sanitize
  436. # update generation so module doesn't get collected
  437. modules[curr].gen = generation
  438. # recurse to imported modules - they might be changed
  439. recursiveDiscovery(modules[curr].imports)
  440. allModulesOrderedByDFS.add(curr)
  441. continue
  442. loadDll(curr.cstring)
  443. # first load all dependencies of the current module and init it after that
  444. recursiveDiscovery(modules[curr].imports)
  445. allModulesOrderedByDFS.add(curr)
  446. modulesToInit.add(curr)
  447. proc initModules() =
  448. # first init the pointers to hcr functions and also do the registering of typeinfo globals
  449. for curr in modulesToInit:
  450. initHcrData(curr.cstring)
  451. initTypeInfoGlobals(curr.cstring)
  452. # for now system always gets fully inited before any other module (including when reloading)
  453. initPointerData(system.cstring)
  454. initGlobalScope(system.cstring)
  455. # proceed with the DatInit calls - for all modules - including the main one!
  456. for curr in allModulesOrderedByDFS:
  457. if curr != system:
  458. initPointerData(curr.cstring)
  459. mainDatInit()
  460. # execute top-level code (in global scope)
  461. for curr in modulesToInit:
  462. if curr != system:
  463. initGlobalScope(curr.cstring)
  464. # cleanup old symbols which are gone now
  465. for curr in modulesToInit:
  466. cleanupSymbols(curr)
  467. proc hcrInit*(moduleList: ptr pointer, main, sys: cstring,
  468. datInit: HcrModuleInitializer, handle: pointer, gpa: HcrProcGetter) {.nimhcr.} =
  469. trace "HCR INITING: ", main.sanitize, " gen: ", generation
  470. # initialize globals
  471. root = $main
  472. system = $sys
  473. mainDatInit = datInit
  474. hcrDynlibHandle = handle
  475. getProcAddr = gpa
  476. # the root is already added and we need it because symbols from it will also be registered in the HCR system
  477. modules[root].imports = getListOfModules(moduleList)
  478. modules[root].gen = high(int) # something huge so it doesn't get collected
  479. # recursively initialize all modules
  480. recursiveDiscovery(modules[root].imports)
  481. initModules()
  482. # the next module to be inited will be the root
  483. currentModule = root
  484. proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} =
  485. let module = hashToModuleMap[moduleHash]
  486. return modules[module].lastModification < getLastModificationTime(module)
  487. proc hcrReloadNeeded*(): bool {.nimhcr.} =
  488. for hash, _ in hashToModuleMap:
  489. if hcrHasModuleChanged(hash):
  490. return true
  491. return false
  492. proc hcrPerformCodeReload*() {.nimhcr.} =
  493. if not hcrReloadNeeded():
  494. trace "HCR - no changes"
  495. return
  496. # We disable the GC during the reload, because the reloading procedures
  497. # will replace type info objects and GC marker procs. This seems to create
  498. # problems when the GC is executed while the reload is underway.
  499. # Future versions of NIMHCR won't use the GC, because all globals and the
  500. # metadata needed to access them will be placed in shared memory, so they
  501. # can be manipulated from external programs without reloading.
  502. when declared(GC_disable):
  503. GC_disable()
  504. defer: GC_enable()
  505. elif declared(GC_disableOrc):
  506. GC_disableOrc()
  507. defer: GC_enableOrc()
  508. inc(generation)
  509. trace "HCR RELOADING: ", generation
  510. var traversedHandlerModules = initHashSet[string]()
  511. proc recursiveExecuteHandlers(isBefore: bool, module: string) =
  512. # do not process an already traversed module
  513. if traversedHandlerModules.containsOrIncl(module): return
  514. traversedHandlerModules.incl module
  515. # first recurse to do a DFS traversal
  516. for curr in modules[module].imports:
  517. recursiveExecuteHandlers(isBefore, curr)
  518. # and then execute the handlers - from leaf modules all the way up to the root module
  519. for curr in modules[module].handlers:
  520. if curr.isBefore == isBefore:
  521. curr.cb()
  522. # first execute the before reload handlers
  523. traversedHandlerModules.clear()
  524. recursiveExecuteHandlers(true, root)
  525. # do the reloading
  526. modulesToInit = @[]
  527. allModulesOrderedByDFS = @[]
  528. recursiveDiscovery(modules[root].imports)
  529. initModules()
  530. # execute the after reload handlers
  531. traversedHandlerModules.clear()
  532. recursiveExecuteHandlers(false, root)
  533. # collecting no longer referenced modules - based on their generation
  534. cleanup modules:
  535. cleanupSymbols(name)
  536. unloadDll(name)
  537. hashToModuleMap.del(modules[name].hash)
  538. modules.del(name)
  539. proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.} =
  540. modules[currentModule].handlers.add(
  541. (isBefore: isBefore, cb: cb))
  542. proc hcrAddModule*(module: cstring) {.nimhcr.} =
  543. if not modules.contains($module):
  544. modules[$module] = newModuleDesc()
  545. proc hcrGeneration*(): int {.nimhcr.} =
  546. generation
  547. proc hcrMarkGlobals*() {.compilerproc, exportc, dynlib, nimcall, gcsafe.} =
  548. # This is gcsafe, because it will be registered
  549. # only in the GC of the main thread.
  550. {.gcsafe.}:
  551. for _, module in modules:
  552. for _, global in module.globals:
  553. if global.markerProc != nil:
  554. global.markerProc()
  555. elif defined(hotcodereloading) or defined(testNimHcr):
  556. when not defined(js):
  557. const
  558. nimhcrLibname = when defined(windows): "nimhcr." & dllExt
  559. elif defined(macosx): "libnimhcr." & dllExt
  560. else: "libnimhcr." & dllExt
  561. {.pragma: nimhcr, compilerproc, importc, dynlib: nimhcrLibname.}
  562. proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.}
  563. proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.}
  564. proc hcrRegisterGlobal*(module: cstring, name: cstring, size: Natural,
  565. gcMarker: HcrGcMarkerProc, outPtr: ptr pointer): bool {.nimhcr.}
  566. proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.}
  567. proc hcrInit*(moduleList: ptr pointer,
  568. main, sys: cstring,
  569. datInit: HcrModuleInitializer,
  570. handle: pointer,
  571. gpa: HcrProcGetter) {.nimhcr.}
  572. proc hcrAddModule*(module: cstring) {.nimhcr.}
  573. proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.}
  574. proc hcrReloadNeeded*(): bool {.nimhcr.}
  575. proc hcrPerformCodeReload*() {.nimhcr.}
  576. proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) {.nimhcr.}
  577. proc hcrMarkGlobals*() {.raises: [], nimhcr, nimcall, gcsafe.}
  578. when declared(nimRegisterGlobalMarker):
  579. nimRegisterGlobalMarker(cast[GlobalMarkerProc](hcrMarkGlobals))
  580. else:
  581. proc hcrHasModuleChanged*(moduleHash: string): bool =
  582. # TODO
  583. false
  584. proc hcrAddEventHandler*(isBefore: bool, cb: proc () {.nimcall.}) =
  585. # TODO
  586. discard