| 1234567891011121314151617181920212223 |
- (define start-state 1)
- (define (tokenize p)
- (let loop ((state start-state)
- (collect (make-collector))
- (i 0)
- (l 0)
- (t '()))
- (cond
- ((equal? (peek-char p) #\tab)
- (error 'tokenize "tab character"))
- ((eof-object? (peek-char p))
- (collector-get collect))
- ((token state (peek-char p)) =>
- (lambda (state)
- (loop state collect i (+ l 1) (cons (read-char p) t))))
- ((accepting? state) =>
- (lambda (emitter)
- (let ((tok (emitter i l (list->string (reverse t)))))
- (when tok (collect-object! collect tok))
- (loop start-state collect (+ i l) 0 '()))))
- (else (error 'tokenize "no parsable token")))))
|
