asc2cld.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. #!/usr/bin/python3
  2. #
  3. # MSX BASIC tokenizer
  4. #
  5. # Copyright © 2020 Pedro Gimeno Fortea
  6. #
  7. #
  8. # Permission is hereby granted, free of charge, to any person obtaining a
  9. # copy of this software and associated documentation files (the "Software"),
  10. # to deal in the Software without restriction, including without limitation
  11. # the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12. # and/or sell copies of the Software, and to permit persons to whom the
  13. # Software is furnished to do so, subject to the following conditions:
  14. #
  15. # The above copyright notice and this permission notice shall be included in
  16. # all copies or substantial portions of the Software.
  17. #
  18. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23. # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  24. # IN THE SOFTWARE.
  25. import sys, re, io, struct
  26. assert sys.hexversion >= 0x3050300, "Invalid Python version, must be 3.5.3+"
  27. tokens1MSX = [
  28. "", "END", "FOR", "NEXT", "DATA", "INPUT", "DIM", "READ", "LET",
  29. "GOTO", "RUN", "IF", "RESTORE", "GOSUB", "RETURN", "REM", "STOP",
  30. "PRINT", "CLEAR", "LIST", "NEW", "ON", "WAIT", "DEF", "POKE", "CONT",
  31. "CSAVE", "CLOAD", "OUT", "LPRINT", "LLIST", "CLS", "WIDTH", "ELSE",
  32. "TRON", "TROFF", "SWAP", "ERASE", "ERROR", "RESUME", "DELETE",
  33. "AUTO", "RENUM", "DEFSTR", "DEFINT", "DEFSNG", "DEFDBL", "LINE",
  34. "OPEN", "FIELD", "GET", "PUT", "CLOSE", "LOAD", "MERGE", "FILES",
  35. "LSET", "RSET", "SAVE", "LFILES", "CIRCLE", "COLOR", "DRAW", "PAINT",
  36. "BEEP", "PLAY", "PSET", "PRESET", "SOUND", "SCREEN", "VPOKE",
  37. "SPRITE", "VDP", "BASE", "CALL", "TIME", "KEY", "MAX", "MOTOR",
  38. "BLOAD", "BSAVE", "DSKO$", "SET", "NAME", "KILL", "IPL", "COPY", "CMD",
  39. "LOCATE", "TO", "THEN", "TAB(", "STEP", "USR", "FN", "SPC(", "NOT",
  40. "ERL", "ERR", "STRING$", "USING", "INSTR", "'", "VARPTR", "CSRLIN",
  41. "ATTR$", "DSKI$", "OFF", "INKEY$", "POINT", ">", "=", "<", "+", "-", "*",
  42. "/", "^", "AND", "OR", "XOR", "EQV", "IMP", "MOD", "\\", "", "", "",
  43. ]
  44. tokens2 = [
  45. "", "LEFT$", "RIGHT$", "MID$", "SGN", "INT", "ABS", "SQR", "RND", "SIN",
  46. "LOG", "EXP", "COS", "TAN", "ATN", "FRE", "INP", "POS", "LEN", "STR$",
  47. "VAL", "ASC", "CHR$", "PEEK", "VPEEK", "SPACE$", "OCT$", "HEX$",
  48. "LPOS", "BIN$", "CINT", "CSNG", "CDBL", "FIX", "STICK", "STRIG", "PDL",
  49. "PAD", "DSKF", "FPOS", "CVI", "CVS", "CVD", "EOF", "LOC", "LOF", "MKI$",
  50. "MKS$", "MKD$", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
  51. "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
  52. "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
  53. "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
  54. ]
  55. trans = (
  56. '\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F'
  57. '\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F'
  58. ' !"#$%&\'()*+,-./0123456789:;<=>?'
  59. '@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_'
  60. '`abcdefghijklmnopqrstuvwxyz{|}~\x7F'
  61. 'ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜ¢£¥₧ƒ'
  62. 'áíóúñѪº¿⌐¬½¼¡«»ÃãĨĩÕõŨũIJij¾∽◊‰¶§'
  63. '▂▚▆���▎▞�����������▘▗▝▖�Δ‡ω█▄▌▐▀'
  64. 'αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■�'
  65. )
  66. # Token parsing modes, affects how integers are parsed
  67. ModeFloat = 0
  68. ModeUint = 1
  69. ModeLiteral = 2
  70. # Create charset translation tables from the above
  71. msx_s2b = {}
  72. msx_b2s = {}
  73. for k, v in enumerate(trans):
  74. b = bytes((k,))
  75. msx_s2b[v] = b
  76. msx_b2s[b] = v
  77. def encode(s):
  78. return b''.join([msx_s2b[c] for c in s])
  79. # Exception for the BASIC errors
  80. class BasicError(Exception):
  81. pass
  82. def num2bytes(s, tok_mode):
  83. """Translate a decimal number string to floating point"""
  84. assert tok_mode != ModeLiteral
  85. assert b'e' not in s, "Lower-case exponent in number"
  86. # Remove embedded spaces and leading zeros
  87. s = s.replace(b' ', b'').lstrip(b'0')
  88. # Unsigned Int is simple
  89. if tok_mode == ModeUint:
  90. return struct.pack('<BH', 0x0E, int(s))
  91. # Find suffix
  92. suffix = s[-1:]
  93. if suffix in {b'%', b'!', b'#'}:
  94. s = s[:-1]
  95. else:
  96. suffix = b''
  97. # Remove point and exponent after taking note of their position and value
  98. point = s.find(b'.')
  99. s = s.replace(b'.', b'')
  100. ep = s.upper().find(b'E') # exponent pointer
  101. exp = 0 # exponent
  102. has_exp = ep != -1
  103. if not has_exp:
  104. ep = len(s)
  105. else:
  106. tmp = s[ep + 1:]
  107. s = s[:ep]
  108. if tmp not in {b'', b'+', b'-'}:
  109. exp = int(tmp, 10)
  110. has_point = point != -1
  111. if point == -1:
  112. point = ep
  113. exp += point + 0x40 # apply bias
  114. # Remove zeros after the decimal point and subtract the count from exp
  115. orig_len = len(s)
  116. s = s.lstrip(b'0')
  117. exp -= orig_len - len(s)
  118. point -= orig_len - len(s)
  119. # Pad with zeros to the right
  120. s += b'0' * (14 - len(s))
  121. # Calculate the integer value for the case of % or no point/exp
  122. nint = 0
  123. if point > 0:
  124. # Perform rounding of the 14th decimal
  125. nint = int(s[:point])
  126. if point <= 14 and (s[point:14] == b'9' * (14 - point)
  127. and s[14:15] >= b'5'):
  128. nint += 1
  129. if suffix == b'%' or (not has_point and not has_exp and suffix == b''
  130. and 0 <= nint <= 32767):
  131. if not (0 <= nint <= 32767):
  132. raise BasicError("Overflow")
  133. if nint < 10:
  134. return struct.pack('<B', 0x11 + nint)
  135. if nint < 256:
  136. return struct.pack('<BB', 0x0F, nint)
  137. return struct.pack('<Bh', 0x1C, nint)
  138. n = int(s[:14])
  139. if suffix == b'!' or (n % 100000000 == 0 and suffix != b'#'):
  140. # Handle as single-precision
  141. n = (n + 50000000) // 100000000
  142. if n > 999999:
  143. # Replicate a bug in MS's tokenizer where 9.999995! is tokenized as 1!
  144. # (uncomment to fix the bug)
  145. #exp += 1
  146. n //= 10
  147. assert 0 <= n <= 999999
  148. if n == 0: exp = 0
  149. if exp > 127 or exp < 0:
  150. raise BasicError("Overflow")
  151. return struct.pack('>BL', 0x1D, int(str(n), 16) + exp * 0x1000000)
  152. if s[14:15] >= b'5':
  153. n += 1
  154. if n > 99999999999999:
  155. exp += 1
  156. n //= 10
  157. assert 0 <= n <= 99999999999999
  158. if n == 0: exp = 0
  159. if exp > 127 or exp < 0:
  160. raise BasicError("Overflow")
  161. return struct.pack('>BLL', 0x1F,
  162. int(str(n // 100000000), 16) + exp * 0x1000000,
  163. int(str(n % 100000000), 16))
  164. def tokenize(src, use_cr=False, type_mode=False, remove_spaces=False):
  165. """Options:
  166. type_mode: True if it should behave like typing in a program; False if
  167. it should behave like LOAD. When True:
  168. - Spaces at EOL are removed.
  169. - Entering line numbers >= 65530 raises Syntax Error.
  170. use_cr: True to split lines at CR and ignore LF (like the real thing),
  171. False to split lines at LF and ignore CR (multiplatform).
  172. remove_spaces: True to "minify" the lines by removing all spaces between
  173. tokens.
  174. """
  175. src = encode(src)
  176. LeadingASCII = re.compile(b'^[A-Z]+')
  177. tok_translate = {}
  178. for k, v in enumerate(tokens1MSX):
  179. v = encode(v)
  180. assert k < 128 and v not in tok_translate
  181. if v != b'':
  182. tok_translate[v] = bytes((k+128,))
  183. for k, v in enumerate(tokens2):
  184. v = encode(v)
  185. assert k < 128 and v not in tok_translate
  186. if v != b'':
  187. tok_translate[v] = bytes((255,k+128))
  188. # Special encoding of ' as :REM'
  189. tok_translate[b"'"] = b':' + tok_translate[b'REM'] + tok_translate[b"'"]
  190. # ? is parsed as PRINT
  191. tok_translate[b'?'] = tok_translate[b'PRINT']
  192. # GOTO admits the spelling GO TO but not e.g. GO TO. GOSUB not affected.
  193. tok_translate[b'GO TO'] = tok_translate[b'GOTO']
  194. # Create list of keywords sorted by longest, escaped
  195. L = []
  196. for v in tokens1MSX + tokens2 + ['GO TO']:
  197. v = encode(v)
  198. if LeadingASCII.search(v):
  199. L.append(v)
  200. L.sort(key=len, reverse=True) # longest match first
  201. # Escape for RE
  202. L = [re.escape(v) for v in L]
  203. # Tokenizer for decimal numbers.
  204. decimal = (
  205. br'|(?P<dec>(?:'
  206. # Cautiously avoid matching 1 space alone or 1 trailing space.
  207. br'(?:[0-9][0-9\ ]*)?\.(?:[0-9\ ]*[0-9])?' # at least a dot
  208. br'|(?:[0-9](?:[0-9\ ]*[0-9])?)' # or at least a digit
  209. br')' # not optional
  210. br'(?:'
  211. br'\ *[%!\#]' # suffix
  212. br'|\ *E\ *[-+]?(?:[0-9\ ]*[0-9])?' # or exponent, but not both
  213. br')?)'
  214. )
  215. # Tokenizer for uint. Up to 6552, read up to 5 digits; from 6553 on, up to 4.
  216. # Used for line numbers, either leading or after GOTO/GOSUB/etc.
  217. uint = (br'|(?P<dec>'
  218. br'(?:0[0\ ]*)?' # leading zeros prefix
  219. br'(?:0' # zero
  220. br'|[1-5](?:\ *[0-9]){,4}' # prefix 1..5999, 5 digits
  221. br'|6\ *[0-4](?:\ *[0-9]){,3}' # prefix 6000..6499, 5 digits
  222. br'|6\ *5\ *[0-4](?:\ *[0-9]){,2}' # prefix 6500..6549, 5 digits
  223. br'|6\ *5\ *5\ *[0-2](?:\ *[0-9])?' # prefix 6550..6552, 5 digits
  224. br'|[6-9](?:\ *[0-9]){,3}' # rest, 4 digits
  225. br'))'
  226. )
  227. tokenizer = (br'(?:'
  228. br"(?P<rem>(?:REM|').*)" # comment
  229. br'|(?P<call>(?:CALL|_)[^:(]*)' # call
  230. br'|(?P<kw>' + br'|'.join(L) + br')' # keywords
  231. br'|[A-Z](?:\ *[0-9.])*' # identifier
  232. br'|&H(?P<hex>[0-9A-F]*)' # hex number
  233. br'|&O(?P<oct>[0-7]*)' # octal number
  234. #br'|&B(?P<bin>[01]+)' # binary numbers don't have tokens
  235. br'%s' # decimal number
  236. br'|(?P<str>"(?:[^"]*)(?:"|$))' # string literal
  237. b'|(?P<del>[\x80-\xFF])' # remove these
  238. br'|.'
  239. br')'
  240. )
  241. tokenizer_uint = re.compile(tokenizer % uint, re.I)
  242. tokenizer_float = re.compile(tokenizer % decimal, re.I)
  243. tokenizer_literal = re.compile(tokenizer % b'|(?P<dec>(?!))', re.I)
  244. del L, uint, decimal, LeadingASCII, tokenizer
  245. # Control Character Behaviour:
  246. # \x00: Terminates the line prematurely (ignores up to the next \x0D).
  247. # \x01-\x08: Skip it and next token.
  248. # \x09: Inserted verbatim, finish token.
  249. # \x0B-\x0C: Replaced by spaces
  250. # \x0E-\x0F,\x11-19,\x1B-\x1F: Inserted verbatim.
  251. # \x0A: Ignored (skipped).
  252. # \x0D: EOL
  253. # \x10: Undefined effects
  254. # Two effects we got:
  255. # - Hang the machine.
  256. # - Delete last byte and insert current line's start address.
  257. # \x1A: EOF. Stops further processing.
  258. # \x7F: Unknown
  259. leadingdigit = re.compile(br'[\ \x01-\x1F\x80-\xFF]*(?:([0-9]?))')
  260. trunc_at_null = re.compile(br'\x00.*', re.S)
  261. call_strip = re.compile(br'[^0-\x7F\ (]*')
  262. # Compile the BASIC to a buffer
  263. buf = io.BytesIO()
  264. # Truncate source at \x1A (^Z)
  265. src = re.sub(b'\x1A.*', b'', src, flags=re.S)
  266. if use_cr:
  267. # realistic
  268. src = src.split(b'\r')
  269. ignore = b'\n'
  270. else:
  271. # practical
  272. src = src.split(b'\n')
  273. ignore = b'\r'
  274. filestart = buf.tell()
  275. # First pass: Read the lines and tokenize them into a dict with the line
  276. # number as the key. Handle line deletion, overwriting, etc.
  277. PRGLines = {}
  278. for line in src:
  279. line = trunc_at_null.sub(b'', line.replace(ignore, b'').lstrip(b' '))
  280. if type_mode: line = line.rstrip(b' ')
  281. if line == b'':
  282. continue
  283. g = tokenizer_uint.match(line)
  284. # Error if not numeric
  285. if not g.group('dec'):
  286. # we can't execute BASIC statements even in typing mode
  287. raise BasicError("Direct statement in file")
  288. p = g.end()
  289. linenum = int(g.group().replace(b' ', b''))
  290. if linenum != 0:
  291. # The first space is not removed for line 0
  292. if line[p:p+1] == b' ':
  293. p += 1
  294. g = leadingdigit.match(line, p)
  295. if type_mode:
  296. # Entering e.g.
  297. # 65530
  298. # alone raises Syntax error. In LOAD mode, it enters 6553 0
  299. if group(1): raise BasicError("Syntax error")
  300. if not g.group(1) and g.end(0) == len(line):
  301. # Delete line
  302. if linenum not in PRGLines:
  303. raise BasicError("Undefined line number")
  304. del PRGLines[linenum]
  305. continue
  306. lbuf = io.BytesIO()
  307. tok_mode = ModeFloat
  308. while True:
  309. nextch = line[p:p+1]
  310. if tok_mode == ModeUint and (nextch == b':'
  311. or b'A' <= nextch <= b'Z'
  312. or b'a' <= nextch <= b'z'):
  313. tok_mode = ModeFloat
  314. if p == len(line): break
  315. tokenizer = tokenizer_float
  316. if tok_mode == ModeUint:
  317. tokenizer = tokenizer_uint
  318. elif tok_mode == ModeLiteral:
  319. tokenizer = tokenizer_literal
  320. g = tokenizer.match(line, p)
  321. assert g, "No match in tokenizer for " + line[p]
  322. match = g.group().upper()
  323. p = g.end()
  324. # Handle control chars... somewhat
  325. if match in {b'\x01', b'\x02', b'\x03', b'\x04', b'\x05', b'\x06',
  326. b'\x07', b'\x08'}:
  327. # Eat a token
  328. g = tokenizer.match(line, p)
  329. p = g.end()
  330. continue
  331. if tok_mode != ModeUint and match in {b'\x0B', b'\x0C', b'\x0E',
  332. b'\x0F', b'\x10', b'\x11', b'\x12', b'\x13', b'\x14', b'\x15',
  333. b'\x16', b'\x17', b'\x18', b'\x19', b'\x1B', b'\x1C', b'\x1D',
  334. b'\x1E', b'\x1F'}:
  335. lbuf.write(b' ')
  336. tok_mode = ModeFloat
  337. continue
  338. # Handle token types
  339. if g.group('rem'):
  340. if g.group().startswith(b"'"):
  341. lbuf.write(tok_translate[b"'"])
  342. lbuf.write(g.group()[1:])
  343. else:
  344. lbuf.write(tok_translate[b'REM'])
  345. lbuf.write(g.group()[3:])
  346. elif g.group('call'):
  347. match = call_strip.sub(b'', match)
  348. if match.startswith(b'_'):
  349. lbuf.write(match)
  350. else: # starts with 'CALL'
  351. lbuf.write(tok_translate[b'CALL'])
  352. lbuf.write(match[4:])
  353. elif g.group('kw'):
  354. # keyword
  355. if match in {b'GOTO', b'GO TO', b'GOSUB', b'THEN', b'ELSE',
  356. b'LIST', b'DELETE', b'AUTO', b'LLIST', b'RENUM', b'RESUME'}:
  357. tok_mode = ModeUint
  358. lbuf.write(tok_translate[match])
  359. elif g.group('hex') is not None:
  360. # hex literal
  361. lbuf.write(b'\x0C')
  362. num = int(g.group('hex'), 16) if g.group('hex') != b'' else 0
  363. if num > 65535:
  364. raise BasicError("Overflow")
  365. lbuf.write(struct.pack('<H', num))
  366. elif g.group('oct') is not None:
  367. # oct literal
  368. lbuf.write(b'\x0B')
  369. num = int(g.group('oct'), 8) if g.group('oct') != b'' else 0
  370. if num > 65535:
  371. raise BasicError("Overflow")
  372. lbuf.write(struct.pack('<H', num))
  373. elif g.group('dec'):
  374. # dec literal
  375. lbuf.write(num2bytes(match, tok_mode))
  376. elif g.group('str'):
  377. # string
  378. lbuf.write(g.group())
  379. elif g.group('del'):
  380. # characters > 127 aren't written
  381. pass
  382. else:
  383. if len(match) == 1 and (b'A' <= match <= b'Z' or b'0' <= match <= b'9'):
  384. tok_mode = ModeLiteral # for identifiers like A1
  385. elif tok_mode == ModeLiteral:
  386. tok_mode = ModeFloat # revert to float mode otherwise
  387. if remove_spaces and match == b' ':
  388. continue
  389. if match in tok_translate:
  390. # some symbols need to be translated to tokens for some reason
  391. lbuf.write(tok_translate[match])
  392. else:
  393. lbuf.write(match)
  394. lbuf.write(b'\0')
  395. PRGLines[linenum] = lbuf.getvalue()
  396. # Second pass - Write remaining lines in order
  397. addr = 0x8001
  398. for linenum in sorted(PRGLines.keys()):
  399. line = PRGLines[linenum]
  400. addr += len(line) + 4
  401. buf.write(struct.pack('<HH', addr, linenum))
  402. buf.write(line)
  403. buf.write(b'\0\0')
  404. return buf.getvalue()
  405. def main():
  406. f = open(sys.argv[1], 'r', newline='')
  407. try:
  408. lines = f.read()
  409. finally:
  410. f.close()
  411. out = tokenize(lines, use_cr=len(sys.argv) > 3 and sys.argv[3] == '1')
  412. f = open(sys.argv[2], 'wb')
  413. try:
  414. f.write(b'\xFF' + out)
  415. finally:
  416. f.close()
  417. if __name__ == '__main__':
  418. main()