Read comments and # literals

1 files changed, 62 insertions, 16 deletions

@@ -43,7 +43,10 @@ function m.inport (source, kind)
                                         end
                                         tok, toktype, self.line =
                                                 m.scan(self.line)()
-                                        if tok then return tok, toktype end
+                                        if tok ~= nil then
+                                                print(tok, toktype)
+                                                return tok, toktype
+                                        end
                                 end
                         end,
         }
@@ -68,20 +71,61 @@ function m.tochars (s)
 end
 
 --- Consumers
--- These take a table of characters (cs) and return a token and the rest of the
--- chars
+-- These take a table of characters (cs) and return:
+-- a token, its type, and the rest of the characters
+
+local token_separator = "[^%s#()\"'`,@;]"
 
 local function consume_token (cs)
         local token = {}
-        while #cs > 0 and cs[1]:match("[^%s()\"#'`,@;]") do
+        while #cs > 0 and cs[1]:match(token_separator) do
                 table.insert(token, pop(cs))
         end
-        return table.concat(token), cs
+        return table.concat(token), "symbol", cs
 end
 
 local function consume_whitespace (cs)
         while #cs > 0 and cs[1]:match("%s") do pop(cs) end
-        return nil, cs
+        return nil, nil, cs
+end
+
+local function consume_comment (cs)
+        local comment = {}
+        repeat table.insert(comment, pop(cs))
+        until #cs == 0 or cs[1]:match("\n")
+        return table.concat(comment), "comment", cs
+end
+
+local function consume_literal (cs)
+        -- whitespace and parantheses character literals.
+        -- reverse the match test b/c it's already a complement
+        if cs[2] == "\\" and not cs[3]:match(token_separator) then
+                return type.character(cs[3])
+        end
+        pop(cs) -- discard '#'
+        local token, value, cs = consume_token(cs) -- todo: vectors #(...)
+        token = "#" .. token -- put '#' back
+        -- tokens!
+        if token == "#t" or token == "#true" then -- booleans
+                value = true
+        elseif token == "#f" or token == "#false" then
+                value = false
+                --[[ To actually *read* nil, I need to change ports from
+                        returning `nil' on eof to an `eof' symbol, i think.
+
+                        elseif token == "#n" or token == "#nil" then
+                        value = nil
+                --]]
+        elseif token == "#\\space" then -- characters
+                value = type.character(" ")
+        elseif token == "#\\newline" then
+                value = type.character("\n")
+        elseif token:match("^#\\") then
+                value = type.character(token:sub(3))
+        else
+                error("Bad literal notation: " .. token)
+        end
+        return value, "literal", cs
 end
 
 --- Reading from a port
@@ -91,7 +135,7 @@ m.readtable = {
         [")"] = function (cs) return pop(cs), "close", cs end,
         ["'"] = function (cs) return pop(cs), "quote", cs end,
         ["`"] = function (cs) return pop(cs), "quote", cs end,
-        [","] =
+        [","] = -- unquote
                 function (cs)
                         pop(cs) -- remove ','
                         if cs[1] == "@" then
@@ -101,8 +145,12 @@ m.readtable = {
                                 return ",", "quote", cs
                         end
                 end,
+        [";"] = consume_comment,
+        ["#"] = consume_literal,
 }
 
+--- TODO: Figure out how to read #f and #n properly
+
 -- Return an iterator over a character table, so you can do:
 -- for token, chars in scan(cs) do ... end
 function m.scan (cs)
@@ -115,12 +163,10 @@ function m.scan (cs)
                                 token, toktype, cs = m.readtable[cs[1]](cs)
                                 return token, toktype, cs
                         elseif cs[1]:match("%s") then
-                                _, cs = consume_whitespace(cs)
-                                return nil, nil, cs
-                                -- return nil, cs
+                                return consume_whitespace(cs)
                         elseif cs[1]:match("[%d.+-]") then
                                 -- numbers, +, -, ., ...
-                                local token, cs = consume_token(cs)
+                                local token, _, cs = consume_token(cs)
                                 if token:match("[-+]") or token == "..." then
                                         return token, "symbol", cs
                                 elseif token == "." then
@@ -131,8 +177,7 @@ function m.scan (cs)
                                         return n, "number", cs
                                 end
                         else
-                                token, cs = consume_token(cs)
-                                return token, "symbol", cs
+                                return consume_token(cs)
                         end
                 end
         end
@@ -143,7 +188,7 @@ function m.readchar (port)
                 local ch = pop(port.line)
                 return ch
         else
-                return port.file.read(1)
+                return port.file and port.file.read(1)
         end
 end
 
@@ -163,11 +208,12 @@ m.readmacros = {
                         table.insert(Q, m.read(port))
                         return t.list(Q)
                 end,
+        comment = function () return nil end,
 }
 
 function m.read (port)
         local function read_ahead (tok, toktype)
-                if not tok then error("Unexpected EOF") end
+                if tok == nil then error("Unexpected EOF") end
                 if toktype == "open" then
                         local L = {}
                         while true do
@@ -192,7 +238,7 @@ function m.read (port)
         end
         -- body of read
         local tok1, toktype1 = port:next_token()
-        if not tok1 then return nil
+        if tok1 == nil then return nil
         else return read_ahead(tok1, toktype1)
         end
 end