--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/progs/lexer/token.sc	Mon Jun 29 21:05:34 2020 +0100
@@ -0,0 +1,56 @@
+// This produces more meaningful tokens and
+// also filters out comments and whitespaces
+//
+// call with
+//
+//  amm token.sc
+//
+
+// load the lexer
+import $file.lexer
+import lexer._ 
+
+
+// The tokens for the WHILE language
+
+abstract class Token 
+case object T_SEMI extends Token
+case object T_LPAREN extends Token
+case object T_RPAREN extends Token
+case class T_ID(s: String) extends Token
+case class T_OP(s: String) extends Token
+case class T_NUM(n: Int) extends Token
+case class T_KWD(s: String) extends Token
+case class T_STR(s: String) extends Token
+
+val token : PartialFunction[(String, String), Token] = {
+  case ("s", _) => T_SEMI
+  case ("p", "{") => T_LPAREN
+  case ("p", "}") => T_RPAREN
+  case ("i", s) => T_ID(s)
+  case ("o", s) => T_OP(s)
+  case ("n", s) => T_NUM(s.toInt)
+  case ("k", s) => T_KWD(s)
+  case ("str", s) => T_STR(s)
+}
+
+// by using collect we filter out all unwanted tokens
+def tokenise(s: String) : List[Token] = 
+  lexing_simp(WHILE_REGS, s).collect(token)
+
+
+
+
+@doc("Tokens for fib and loops programs.")
+@main
+def main() = {
+  println("Fib program")
+  println(tokenise(prog2))
+  println("Loops program")
+  println(tokenise(prog3))
+
+  for (i <- 0 to 20 by 5) {
+    println(f"$i%2.0f: ${time(tokenise(prog3 * i))._2}")
+  }
+
+}
\ No newline at end of file