Jagna/tokenizer.ml

open Printf
open Location
open Utf8


type token = | Lpar | Rpar | Str of string
type located_token = token * location


type pos_tracking_stream = char Stream.t * (position ref) * (int ref)
let current_position (_, pos, _) = !pos
let init_tracking_stream stream = stream , (ref (0,0)) , ref 0
let peek (stream, _, _) = Stream.peek stream
let junk (stream, pos, unc (*until-next-char*) ) : unit =  let (row, col) = !pos in
  (match Stream.peek stream with
   | Some c ->
     if !unc > 0 then unc := !unc - 1 else
       begin match c with
         | '\n' ->  pos := (row+1, 0) ; unc := 0
         | c -> unc := (char_length c) - 1; pos := (row, col+1)
       end
   | None -> ());
  Stream.junk stream
    
let fetch_in_stream file_name = init_tracking_stream (Stream.of_channel (open_in file_name))

let rec generate_token tracking_stream : located_token option = match peek tracking_stream with
  | None -> None
  | Some c -> let cp = (current_position tracking_stream) in begin match c with
      | '(' -> junk tracking_stream; Some (Lpar, Single cp)
      | ')' -> junk tracking_stream; Some (Rpar, Single cp)
      | ' ' | '\n' -> junk tracking_stream; generate_token tracking_stream
      | _ ->
        let rec parse_atom str = match peek tracking_stream with
          | None -> Str str
          | Some xd -> begin match xd with
              |  ')' | '(' |  ' ' | '\n' -> Str str
              | o -> junk tracking_stream ; parse_atom (str ^ (String.make 1 o))
            end
        in
        let atom = parse_atom "" in
        Some (atom, Range (cp, back_one_char (current_position tracking_stream)))
    end

let tokenize char_stream : located_token Stream.t = Stream.from (fun _ -> (generate_token char_stream))

let sprintf_token  = function
  |Lpar -> "("
  |Rpar -> ")"
  |Str s -> s
    
let print_token (token, location) =
  printf "%s: %s\n" (sprintf_token token) (sprintf_location location)
                                    
(*let main = let stream = fetch_in_stream "/tmp/xd" in
  let tokens = tokenize stream in
  Stream.iter print_token tokens*)