sourcemap.nim 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208
  1. import std/[strutils, strscans, parseutils, assertions]
  2. type
  3. Segment = object
  4. ## Segment refers to a block of something in the JS output.
  5. ## This could be a token or an entire line
  6. original: int # Column in the Nim source
  7. generated: int # Column in the generated JS
  8. name: int # Index into names list (-1 for no name)
  9. Mapping = object
  10. ## Mapping refers to a line in the JS output.
  11. ## It is made up of segments which refer to the tokens in the line
  12. case inSource: bool # Whether the line in JS has Nim equivilant
  13. of true:
  14. file: int # Index into files list
  15. line: int # 0 indexed line of code in the Nim source
  16. segments: seq[Segment]
  17. else: discard
  18. SourceInfo = object
  19. mappings: seq[Mapping]
  20. names, files: seq[string]
  21. SourceMap* = object
  22. version*: int
  23. sources*: seq[string]
  24. names*: seq[string]
  25. mappings*: string
  26. file*: string
  27. func addSegment(info: var SourceInfo, original, generated: int, name: string = "") {.raises: [].} =
  28. ## Adds a new segment into the current line
  29. assert info.mappings.len > 0, "No lines have been added yet"
  30. var segment = Segment(original: original, generated: generated, name: -1)
  31. if name != "":
  32. # Make name be index into names list
  33. segment.name = info.names.find(name)
  34. if segment.name == -1:
  35. segment.name = info.names.len
  36. info.names &= name
  37. assert info.mappings[^1].inSource, "Current line isn't in Nim source"
  38. info.mappings[^1].segments &= segment
  39. func newLine(info: var SourceInfo) {.raises: [].} =
  40. ## Add new mapping which doesn't appear in the Nim source
  41. info.mappings &= Mapping(inSource: false)
  42. func newLine(info: var SourceInfo, file: string, line: int) {.raises: [].} =
  43. ## Starts a new line in the mappings. Call addSegment after this to add
  44. ## segments into the line
  45. var mapping = Mapping(inSource: true, line: line)
  46. # Set file to file position. Add in if needed
  47. mapping.file = info.files.find(file)
  48. if mapping.file == -1:
  49. mapping.file = info.files.len
  50. info.files &= file
  51. info.mappings &= mapping
  52. # base64_VLQ
  53. func encode*(values: seq[int]): string {.raises: [].} =
  54. ## Encodes a series of integers into a VLQ base64 encoded string
  55. # References:
  56. # - https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/
  57. # - https://github.com/rails/sprockets/blob/main/guides/source_maps.md#source-map-file
  58. const
  59. alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
  60. shift = 5
  61. continueBit = 1 shl 5
  62. mask = continueBit - 1
  63. for val in values:
  64. # Sign is stored in first bit
  65. var newVal = abs(val) shl 1
  66. if val < 0:
  67. newVal = newVal or 1
  68. # Now comes the variable length part
  69. # This is how we are able to store large numbers
  70. while true:
  71. # We only encode 5 bits.
  72. var masked = newVal and mask
  73. newVal = newVal shr shift
  74. # If there is still something left
  75. # then signify with the continue bit that the
  76. # decoder should keep decoding
  77. if newVal > 0:
  78. masked = masked or continueBit
  79. result &= alphabet[masked]
  80. # If the value is zero then we have nothing left to encode
  81. if newVal == 0:
  82. break
  83. iterator tokenize*(line: string): (int, string) =
  84. ## Goes through a line and splits it into Nim identifiers and
  85. ## normal JS code. This allows us to map mangled names back to Nim names.
  86. ## Yields (column, name). Doesn't yield anything but identifiers.
  87. ## See mangleName in compiler/jsgen.nim for how name mangling is done
  88. var
  89. col = 0
  90. token = ""
  91. while col < line.len:
  92. var
  93. token: string
  94. name: string
  95. # First we find the next identifier
  96. col += line.skipWhitespace(col)
  97. col += line.skipUntil(IdentStartChars, col)
  98. let identStart = col
  99. col += line.parseIdent(token, col)
  100. # Idents will either be originalName_randomInt or HEXhexCode_randomInt
  101. if token.startsWith("HEX"):
  102. var hex: int
  103. # 3 = "HEX".len and we only want to parse the two integers after it
  104. discard token[3 ..< 5].parseHex(hex)
  105. name = $chr(hex)
  106. elif not token.endsWith("_Idx"): # Ignore address indexes
  107. # It might be in the form originalName_randomInt
  108. let lastUnderscore = token.rfind('_')
  109. if lastUnderscore != -1:
  110. name = token[0..<lastUnderscore]
  111. if name != "":
  112. yield (identStart, name)
  113. func parse*(source: string): SourceInfo =
  114. ## Parses the JS output for embedded line info
  115. ## So it can convert those into a series of mappings
  116. var
  117. skipFirstLine = true
  118. currColumn = 0
  119. currLine = 0
  120. currFile = ""
  121. # Add each line as a node into the output
  122. for line in source.splitLines():
  123. var
  124. lineNumber: int
  125. linePath: string
  126. column: int
  127. if line.strip().scanf("/* line $i:$i \"$+\" */", lineNumber, column, linePath):
  128. # When we reach the first line mappinsegmentg then we can assume
  129. # we can map the rest of the JS lines to Nim lines
  130. currColumn = column # Column is already zero indexed
  131. currLine = lineNumber - 1
  132. currFile = linePath
  133. # Lines are zero indexed
  134. result.newLine(currFile, currLine)
  135. # Skip whitespace to find the starting column
  136. result.addSegment(currColumn, line.skipWhitespace())
  137. elif currFile != "":
  138. result.newLine(currFile, currLine)
  139. # There mightn't be any tokens so add a starting segment
  140. result.addSegment(currColumn, line.skipWhitespace())
  141. for jsColumn, token in line.tokenize:
  142. result.addSegment(currColumn, jsColumn, token)
  143. else:
  144. result.newLine()
  145. func toSourceMap*(info: SourceInfo, file: string): SourceMap {.raises: [].} =
  146. ## Convert from high level SourceInfo into the required SourceMap object
  147. # Add basic info
  148. result.version = 3
  149. result.file = file
  150. result.sources = info.files
  151. result.names = info.names
  152. # Convert nodes into mappings.
  153. # Mappings are split into blocks where each block referes to a line in the outputted JS.
  154. # Blocks can be seperated into statements which refere to tokens on the line.
  155. # Since the mappings depend on previous values we need to
  156. # keep track of previous file, name, etc
  157. var
  158. prevFile = 0
  159. prevLine = 0
  160. prevName = 0
  161. prevNimCol = 0
  162. for mapping in info.mappings:
  163. # We know need to encode segments with the following fields
  164. # All these fields are relative to their previous values
  165. # - 0: Column in generated code
  166. # - 1: Index of Nim file in source list
  167. # - 2: Line in Nim source
  168. # - 3: Column in Nim source
  169. # - 4: Index in names list
  170. if mapping.inSource:
  171. # JS Column is special in that it is reset after every line
  172. var prevJSCol = 0
  173. for segment in mapping.segments:
  174. var values = @[segment.generated - prevJSCol, mapping.file - prevFile, mapping.line - prevLine, segment.original - prevNimCol]
  175. # Add name field if needed
  176. if segment.name != -1:
  177. values &= segment.name - prevName
  178. prevName = segment.name
  179. prevJSCol = segment.generated
  180. prevNimCol = segment.original
  181. prevFile = mapping.file
  182. prevLine = mapping.line
  183. result.mappings &= encode(values) & ","
  184. # Remove trailing ,
  185. if mapping.segments.len > 0:
  186. result.mappings.setLen(result.mappings.len - 1)
  187. result.mappings &= ";"
  188. proc genSourceMap*(source: string, outFile: string): SourceMap =
  189. let node = parse(source)
  190. result = node.toSourceMap(outFile)