Procker/pre-process.scm

(import
  (scheme base)
  (scheme read)
  (scheme write))

(define symbol-terminators
  '(#\space #\newline #\tab
    #\. #\; #\= #\+ #\* #\/ #\-
    #\( #\) #\, #\< #\> #\[ #\]
    #\{ #\}))

(define keywords
  '(break case continue else for if return while))

(define operators
  '(+ - * / = += -= *= /= < > == != <= >=))

(define (symbol-terminator? char)
  (not (not (memv char symbol-terminators))))

(define (keyword? sym)
  (not (not (memq sym keywords))))

(define (operator? sym)
  (not (not (memq sym operators))))

(define (symbol-start? char)
  (define i (char->integer char))
  (or (<= 48 i 57)
      (<= 65 i 90)
      (<= 97 i 122)))

(define (tokenize-expression str)
  (define p (open-input-string str))
  (let loop ((result '())
             (next-char (peek-char p)))
    (if (eof-object? next-char)
        (begin
          (close-input-port p)
          (reverse result))
        (cond
         ((symbol-start? next-char)
          (let ((sym (parse-symbol p)))
            (loop (cons sym result)
                  (peek-char p))))
         (else
          ;; to do
          (read-char p)
          (loop result
                (peek-char p)))))))